[Refactor] Translate each KV loc once per iteration (#42753)

This commit is contained in:
Cheng Wan
2026-10-08 00:58:28 -07:00
committed by GitHub
parent 508ddc859a
commit b3e6c87be3
97 changed files with 3393 additions and 3289 deletions
@@ -30,10 +30,11 @@ Two delivery forms over that one formula:
which the pool bounds, where a page table's width is bounded
only by `max_context_len`.
PREFIX-ONLY per row: nothing past the row's live prefix is written, so a
caller-owned buffer keeps what it had there -- which is what lets a captured
cuda-graph buffer be refreshed in place. Readers bound themselves by
`cache_seqlens` and never look past the prefix.
PREFIX-ONLY per row by default: nothing past the row's live prefix is written,
so a caller-owned buffer keeps what it had there -- which is what lets a
captured cuda-graph buffer be refreshed in place. Readers bound themselves by
`cache_seqlens` and never look past the prefix. A fresh (`torch.empty`) table
asks for the sink past the prefix instead (`zero_tail`).
A `-1` in `req_to_token` and a freed (`-1`) v2p row both clamp to entry 0, the
reserved padding slot, so a kernel dereferences padding, not a wild address.
@@ -60,6 +61,56 @@ _TARGET_BLOCKS = 1024
@triton.jit
def _gather_translate_items(
row_in, # this row of `req_to_token` -- VIRTUAL token ids
row_out, # where this row's items land
v2p_ptr, # [num_pages + 1] int64 -- virtual->physical page table
live, # scalar: the row reads `req_to_token`; else every item is the sink
kv_start, # first token of the row's window
n_items, # live items in the row
n_written, # items stored: `n_items`, or a wider zeroed row
item_begin, # this program's first item
item_stride, # items one program advances per loop trip
PAGE_SIZE: tl.constexpr,
EMIT_PER_TOKEN: tl.constexpr,
OUT_INT64: tl.constexpr,
ZERO_TAIL: tl.constexpr,
BLOCK: tl.constexpr,
):
for start in range(item_begin, n_written, item_stride):
item = start + tl.arange(0, BLOCK)
mask = item < n_items
load_mask = mask & live
pos = kv_start + item
if EMIT_PER_TOKEN:
page = pos // PAGE_SIZE
else:
page = pos
tok = tl.load(
row_in + page.to(tl.int64) * PAGE_SIZE, mask=load_mask, other=0
).to(tl.int64)
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
# -1 at ps == 1, which would read one element BEFORE `v2p`.
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=load_mask, other=0), 0)
if EMIT_PER_TOKEN:
value = entry * PAGE_SIZE + pos % PAGE_SIZE
else:
value = entry
store_mask = mask
if ZERO_TAIL:
value = tl.where(mask, value, 0)
store_mask = item < n_written
if OUT_INT64:
tl.store(row_out + item, value, mask=store_mask)
else:
tl.store(row_out + item, value.to(tl.int32), mask=store_mask)
# The per-call widths and lengths stay unspecialized: Triton would otherwise
# compile a new variant (`== 1`, `% 16`) mid-serving each time a batch's shape
# hits a combination it has not seen.
@triton.jit(do_not_specialize=["out_stride", "seq_len_delta", "max_row_items"])
def build_kv_read_indices_kernel(
req_to_token_ptr, # in: [max_reqs, max_context] -- VIRTUAL token ids
req_pool_indices_ptr, # in: [bs] -- req_to_token row per batch lane
@@ -76,6 +127,7 @@ def build_kv_read_indices_kernel(
PAGE_SIZE: tl.constexpr,
EMIT_PER_TOKEN: tl.constexpr,
OUT_INT64: tl.constexpr,
ZERO_TAIL: tl.constexpr,
BLOCK: tl.constexpr,
):
bid = tl.program_id(0)
@@ -98,30 +150,24 @@ def build_kv_read_indices_kernel(
# A widened window can outrun a fixed-width row (a verify near the
# context limit); writing past it would land in the next row.
n_items = tl.minimum(n_items, max_row_items)
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
item = start + tl.arange(0, BLOCK)
mask = item < n_items
pos = kv_start + item
if EMIT_PER_TOKEN:
page = pos // PAGE_SIZE
else:
page = pos
tok = tl.load(row_in + page.to(tl.int64) * PAGE_SIZE, mask=mask, other=0).to(
tl.int64
)
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
# -1 at ps == 1, which would read one element BEFORE `v2p`.
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=mask, other=0), 0)
if EMIT_PER_TOKEN:
value = entry * PAGE_SIZE + pos % PAGE_SIZE
else:
value = entry
if OUT_INT64:
tl.store(row_out + item, value, mask=mask)
else:
tl.store(row_out + item, value.to(tl.int32), mask=mask)
# A fresh (unzeroed) table also takes the sink past the live prefix.
n_written = max_row_items if ZERO_TAIL else n_items
_gather_translate_items(
row_in,
row_out,
v2p_ptr,
True,
kv_start,
n_items,
n_written,
tl.program_id(1) * BLOCK,
item_stride,
PAGE_SIZE=PAGE_SIZE,
EMIT_PER_TOKEN=EMIT_PER_TOKEN,
OUT_INT64=OUT_INT64,
ZERO_TAIL=ZERO_TAIL,
BLOCK=BLOCK,
)
def _launch(
@@ -138,6 +184,7 @@ def _launch(
kv_start_idx: Optional[torch.Tensor],
emit_per_token: bool,
seq_len_delta: int = 0,
zero_tail: bool = False,
) -> None:
bs = int(req_pool_indices.numel())
item_programs = min(
@@ -159,6 +206,7 @@ def _launch(
PAGE_SIZE=page_size,
EMIT_PER_TOKEN=emit_per_token,
OUT_INT64=out.dtype == torch.int64,
ZERO_TAIL=zero_tail,
BLOCK=_BLOCK_ITEMS,
num_warps=_NUM_WARPS,
)
@@ -188,17 +236,18 @@ def build_kv_read_table(
max_pages: int,
out: torch.Tensor,
seq_len_delta: int = 0,
zero_tail: bool = False,
) -> torch.Tensor:
"""Fill ``out``'s live prefix with PAGE TABLE entries.
``out`` is caller-owned (fresh zeros for the eager path, the module's
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
region's live prefix is written -- never rebound, never tail-cleared.
``out`` is caller-owned and only its ``[:bs, :max_pages]`` region is
written: each row's live prefix, and with ``zero_tail`` the sink (0) past
it up to ``max_pages`` -- what a fresh ``torch.empty`` table needs, in the
same launch. Never rebound.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). Every row stops at
``max_pages``: a widened window that outruns the row is cut there rather
than spilling into the next row, on CUDA as on CPU.
``seq_len_delta`` widens every row's live prefix (a verify reads its draft
tokens' KV back from the pool); a widened row is cut at ``max_pages``
rather than spilling into the next row.
"""
bs = int(req_pool_indices.numel())
assert out.dtype == torch.int32, (
@@ -228,6 +277,8 @@ def build_kv_read_table(
v2p=v2p,
page_size=page_size,
).to(torch.int32)
if zero_tail:
out[b, live:max_pages] = 0
return out
_launch(
@@ -243,6 +294,7 @@ def build_kv_read_table(
kv_start_idx=None,
emit_per_token=False,
seq_len_delta=seq_len_delta,
zero_tail=zero_tail,
)
return out
@@ -89,6 +89,29 @@ def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
@triton.jit
def _load_token_ids(
row_ptr,
pos,
mask,
v2p,
page_size: tl.constexpr,
ENTRY_PAGE_SIZE: tl.constexpr,
TRANSLATE: tl.constexpr,
):
"""Token ids at positions ``pos`` of one read-source row: the row's own
entries when token-granular, ``entry * ps + pos % ps`` when page-granular;
with ``TRANSLATE``, virtual ids mapped through ``v2p`` on the way out."""
if ENTRY_PAGE_SIZE == 1:
ids = tl.load(row_ptr + pos, mask=mask)
else:
entry = tl.load(row_ptr + pos // ENTRY_PAGE_SIZE, mask=mask, other=0)
ids = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
if TRANSLATE:
ids = _translate_token_ids(ids, v2p, mask, page_size)
return ids
@triton.jit
def generate_draft_decode_kv_indices(
req_pool_indices,
@@ -97,8 +120,9 @@ def generate_draft_decode_kv_indices(
kv_indices,
kv_indptr,
positions,
v2p,
pool_len: tl.constexpr,
# Runtime, not constexpr: the unified pool's read table is as wide as the
# iteration's longest row, so a constexpr stride would recompile per width.
row_stride,
kv_indices_stride: tl.constexpr,
kv_indptr_stride: tl.constexpr,
bs_upper: tl.constexpr,
@@ -108,6 +132,9 @@ def generate_draft_decode_kv_indices(
window_size: tl.constexpr = 0,
sink_size: tl.constexpr = 0,
NUM_STEPS: tl.constexpr = 0,
ENTRY_PAGE_SIZE: tl.constexpr = 1,
# The source's page table (`KVIndexTable.v2p`), applied with TRANSLATE.
v2p=None,
TRANSLATE: tl.constexpr = False,
):
# window_size > 0 restricts the draft (not the target) to sink_size prefix
@@ -151,7 +178,7 @@ def generate_draft_decode_kv_indices(
# Update kv_indices
kv_offset = cum_seq_len * topk + bid * iters * topk + topk_id * (seq_len_w + iters)
kv_ptr = kv_indices + kv_offset
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * pool_len
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
num_loop = tl.cdiv(seq_len_w, BLOCK_SIZE)
if NUM_STEPS != 0 and blk >= num_loop and blk > 0:
@@ -165,9 +192,9 @@ def generate_draft_decode_kv_indices(
copy_offset,
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
data = _load_token_ids(
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
)
tl.store(kv_ptr + copy_offset, data, mask=mask)
copy_offset += BLOCK_SIZE
else:
@@ -179,9 +206,9 @@ def generate_draft_decode_kv_indices(
copy_offset,
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
data = _load_token_ids(
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
)
tl.store(kv_ptr + copy_offset, data, mask=mask)
# Extension entries and kv_indptr belong to token block 0 alone; other
@@ -189,12 +216,14 @@ def generate_draft_decode_kv_indices(
if blk == 0:
extend_offset = tl.arange(0, iter_upper)
if page_size == 1 or topk == 1:
extend_data = tl.load(
token_pool_ptr
+ seq_len
+ topk_id * num_steps
+ tl.arange(0, iter_upper),
mask=extend_offset < iters,
extend_data = _load_token_ids(
token_pool_ptr,
seq_len + topk_id * num_steps + tl.arange(0, iter_upper),
extend_offset < iters,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
else:
prefix_len = seq_len
@@ -208,14 +237,14 @@ def generate_draft_decode_kv_indices(
+ topk_id * num_new_pages_per_topk * page_size
+ last_page_len
)
extend_data = tl.load(
token_pool_ptr + start + extend_offset,
mask=extend_offset < iters,
)
if TRANSLATE:
extend_data = _translate_token_ids(
extend_data, v2p, extend_offset < iters, page_size
extend_data = _load_token_ids(
token_pool_ptr,
start + extend_offset,
extend_offset < iters,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
tl.store(
@@ -20,6 +20,9 @@ class RaggedVerifyWindow(msgspec.Struct, frozen=True):
positions: torch.Tensor
verify_cache_loc: torch.Tensor
verify_ids: torch.Tensor
# Each packed lane's flat position in the `[bs, verify_num_draft_tokens]`
# verify window, -1 on a padded lane.
window_index: torch.Tensor
class BuildRaggedVerifyWindow:
@@ -129,6 +132,9 @@ def build_ragged_verify_window(
positions=positions,
verify_cache_loc=verify_cache_loc,
verify_ids=verify_ids,
window_index=torch.where(
valid, safe_req.to(torch.int64) * verify_num_draft_tokens + within, -1
),
)
@@ -176,7 +182,7 @@ def build_ragged_verify_window_triton(
verify_lens = layout.verify_lens.to(device=device, dtype=torch.int32)
padded_total = layout.graph_num_tokens
req_id, within, _valid = compact_row_index_triton(
req_id, within, valid = compact_row_index_triton(
verify_lens=verify_lens, padded_total=padded_total, device=device
)
real_cache_loc = assign_extend_cache_locs_func(
@@ -218,6 +224,11 @@ def build_ragged_verify_window_triton(
positions=positions,
verify_cache_loc=verify_cache_loc,
verify_ids=verify_ids,
window_index=torch.where(
valid,
req_id.clamp(max=bs - 1).to(torch.int64) * verify_num_draft_tokens + within,
-1,
),
)
+9 -14
View File
@@ -499,10 +499,9 @@ def _assert_spec_verify_backends(
) -> None:
"""Refuse target backends whose verify id rails are not translation-audited.
Only the target's prefill/decode pair is checked here. A draft that fuses
into the target's pages must run on the translated rails; the fused-draft
decision enforces that by declining fusion otherwise, and a private-pool
draft indexes its own pool by virtual id."""
Only the target's prefill/decode pair is checked here: fusion declines for
a draft off the translated rails, and a private-pool draft indexes its own
pool by virtual id."""
if allowed is None:
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
dcp_note = ""
@@ -591,8 +590,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"DFLASH",
), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK (chain draft), DFLASH (fused block draft), and "
"EAGLE/EAGLE3 (fused draft KV); other speculative algorithms are "
"DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are "
"not yet audited for the unified pool's virtual-to-physical loc "
"translation. Got "
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
@@ -627,9 +625,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
f"spec-verify-audited attention backends {sorted(eagle_allowed)} "
f"(got {sorted(eagle_backends, key=str)})."
)
# An unset draft backend inherits the target's pair. A fused draft is
# MHA-shaped on every host, and these rails also serve a draft that
# keeps an MLA pool of its own.
# An unset draft backend inherits the target's pair.
draft_backend = cfg.speculative_draft_attention_backend
draft_backends = {draft_backend} if draft_backend else eagle_backends
assert draft_backends <= TRANSLATED_MHA_RAILS, (
@@ -649,9 +645,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
)
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
"TBO's replay split hands each child a view without the pre-translate "
"write loc, so a captured decode replay raises. "
"TODO(ch-wan): carry out_cache_loc_virtual into the child view."
"the iteration's read tables are indexed by the whole batch's rows, and "
"a TBO child's readers do not offset into them."
)
assert not cfg.enable_lmcache, (
"--enable-unified-memory is not yet compatible with --enable-lmcache: "
@@ -679,8 +674,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
), (
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
"--disaggregation-decode-retraction-backup=host_pool: the backup builds "
"the draft's host pool off a device pool of its own, and an EAGLE draft "
"on the unified pool lives fused in the target's pages, without one."
"the draft's host pool off the draft's own device pool, which a fused "
"EAGLE draft does not have."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
@@ -698,6 +698,14 @@ class TboForwardBatchPreparer:
start_token_index:end_token_index
]
output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical
# Unified memory refuses two-batch overlap, so the plan's reads stay in
# `req_to_token`, which each child indexes at its own rows; each child
# writes its own tokens of the plan's window.
output_dict["kv_loc_plan"] = batch.kv_loc_plan
if batch.kv_loc_plan is not None:
output_dict["kv_loc_cols"] = batch.kv_loc_plan.cols_slice(
batch.kv_loc_cols, slice(start_token_index, end_token_index)
)
attention_tp_size = get_parallel().attn_tp_size
_tbo_padded_len = (
+4 -12
View File
@@ -18,6 +18,7 @@ from sglang.srt.layers.attention.triton_backend import (
ForwardMetadata,
update_sliding_window_buffer,
)
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_capture_attention_variant,
@@ -202,12 +203,10 @@ class DllmFlashAttention:
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
if backend.use_sliding_window_kv_pool
else None,
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
if backend.kv_index_translator.is_translating
else None,
)
metadata = self._prefill_graph_metadata[key]
backend._fill_kv_indptr_and_indices(
batch.kv_loc_plan,
bs,
batch.extend_prefix_lens,
batch.req_pool_indices,
@@ -220,6 +219,7 @@ class DllmFlashAttention:
_, _, _, offsets = update_sliding_window_buffer(
metadata.window_kv_indptr,
backend.kv_index_translator,
batch.kv_loc_plan,
batch.req_pool_indices,
backend.sliding_window_size,
batch.extend_prefix_lens,
@@ -231,15 +231,7 @@ class DllmFlashAttention:
metadata.window_kv_offsets.copy_(offsets)
if metadata.swa_out_cache_loc is not None:
metadata.swa_out_cache_loc.copy_(
backend.kv_index_translator.sliding_window_write_loc_for(
batch.out_cache_loc
)
)
if metadata.out_cache_loc_full_physical is not None:
backend.kv_index_translator.fill_capture_write_loc(
out=metadata.out_cache_loc_full_physical,
forward_batch=batch,
width=metadata.out_cache_loc_full_physical.numel(),
backend.kv_index_translator.write_ids(batch, IdSpaceKind.SLIDING_WINDOW)
)
backend.forward_metadata = metadata
@@ -40,6 +40,7 @@ from sglang.srt.utils import is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -1951,6 +1952,7 @@ class AiterAttnBackend(AttentionBackend):
paged_kernel_lens=forward_batch.seq_lens,
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
translator=self.kv_index_translator,
plan=forward_batch.kv_loc_plan,
)
)
self.forward_metadata = ForwardMetadata(
@@ -2150,6 +2152,7 @@ class AiterAttnBackend(AttentionBackend):
max(forward_batch.extend_seq_lens_cpu),
forward_batch.seq_lens_cpu.max().item(),
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
max_q_len = self.mla_indices_updater_prefill.max_q_len
@@ -2190,6 +2193,7 @@ class AiterAttnBackend(AttentionBackend):
prefix_lens,
encoder_lens=forward_batch.encoder_lens,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
if self.use_sliding_window_kv_pool:
@@ -4454,6 +4458,8 @@ class AiterIndicesUpdaterPrefill:
prefix_lens: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -4466,6 +4472,8 @@ class AiterIndicesUpdaterPrefill:
prefix_lens: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
kv_start_idx = None
kv_indptr = self.kv_indptr
@@ -4515,6 +4523,7 @@ class AiterIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.kv_index_translator,
plan=plan,
)
)
@@ -4547,6 +4556,8 @@ class AiterMlaIndicesUpdaterPrefill:
max_q_len: int,
max_kv_len: int,
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -4560,6 +4571,8 @@ class AiterMlaIndicesUpdaterPrefill:
max_q_len: int,
max_kv_len: int,
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
bs = len(req_pool_indices)
@@ -4594,6 +4607,7 @@ class AiterMlaIndicesUpdaterPrefill:
paged_kernel_lens=kv_lens,
paged_kernel_lens_sum=kv_lens_sum,
translator=self.kv_index_translator,
plan=plan,
)
)
@@ -4663,18 +4677,21 @@ class AiterMultiStepDraftBackend:
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
else 1
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
triton.next_power_of_2(num_seqs),
@@ -4684,7 +4701,9 @@ class AiterMultiStepDraftBackend:
# A single token block is the historical launch; NUM_STEPS=0 keeps
# its 128-wide program instead of the token-block specialization.
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
for i in range(self.speculative_num_steps - 1):
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_flashinfer_available
@@ -176,7 +177,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
query = self._set_kv_and_concat_q_fp8_fused(
layer=layer,
@@ -208,7 +209,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
assert k is not None and k_rope is not None
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
self._kv_write_loc(forward_batch),
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
@@ -33,7 +33,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -54,6 +54,7 @@ from sglang.srt.utils import get_compiler_backend
from sglang.srt.utils.common import get_device_capability
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -466,10 +467,9 @@ class FlashAttentionBackend(AttentionBackend):
if not forward_batch.forward_mode.is_draft_extend_v2():
return
if self.kv_index_translator.is_translating:
# The draft-extend runners record this call unconditionally, but a
# translating backend rebuilds these tables out of graph on every
# replay; a captured raw req_to_token gather would then overwrite
# them with virtual ids.
# The draft-extend runners record this call unconditionally; a
# translating backend rebuilds these tables out of graph, and a
# captured req_to_token gather would overwrite them with virtual ids.
return
bs = forward_batch.batch_size
metadata = self.draft_extend_metadata[bs]
@@ -543,7 +543,11 @@ class FlashAttentionBackend(AttentionBackend):
self._init_full_cg_decode_metadata(forward_batch, in_capture)
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
locations = getattr(forward_batch, "out_cache_loc", None)
"""This step's sliding-window write ids: the plan's, sliced per
draft step the way `out_cache_loc` is."""
locations = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
if (
locations is not None
and forward_batch.forward_mode.is_decode_or_idle()
@@ -586,11 +590,7 @@ class FlashAttentionBackend(AttentionBackend):
if in_capture:
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
)
)
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
if in_capture:
num_tokens = forward_batch.positions.numel()
@@ -627,6 +627,7 @@ class FlashAttentionBackend(AttentionBackend):
return
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -678,6 +679,7 @@ class FlashAttentionBackend(AttentionBackend):
# A stale non-None seq_lens_cpu buffer would under-size max_seq_pages
# (stale page-table rows -> OOB); force None under sync-free.
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -755,22 +757,17 @@ class FlashAttentionBackend(AttentionBackend):
translating = self.kv_index_translator.is_translating
max_seq_len_k = int(forward_batch.seq_lens_cpu[:bs].max().item())
if translating:
# Unified pool: the block table is a TRANSLATED page table, built
# straight into these capture-stable buffers from the LIVE v2p, so
# a page relocated by compaction since capture is picked up. Same
# substitution the eager extend branch makes in its `_unified_read`
# fixup; `build_index_table` emits page-granular physical ids
# directly, so there is no `// page_size` to undo.
self.kv_index_translator.build_index_table(
req_pool_indices=forward_batch.req_pool_indices[:bs],
seq_lens=forward_batch.seq_lens[:bs],
into=KVReadTables(
full=m.page_table,
sliding_window=(
m.swa_page_table if self.use_sliding_window_kv_pool else None
),
),
# Unified pool: the plan's page table, physical page ids from this
# iteration's v2p, copied into these capture-stable buffers.
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=m.page_table
)
if self.use_sliding_window_kv_pool:
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan,
out=m.swa_page_table,
kind=IdSpaceKind.SLIDING_WINDOW,
)
elif max_seq_len_k > 0:
# Build the block table like the eager extend branch: take every
# page_size-th token slot from req_to_token and divide by page_size.
@@ -797,22 +794,15 @@ class FlashAttentionBackend(AttentionBackend):
self.full_cg_prefill_swa_out_cache_loc.shape[0],
"full-CG prefill SWA write-location buffer",
)
# Under the unified pool `out_cache_loc` was rebound to FULL-side
# physical ids at ForwardBatch construction, so the full->swa map
# cannot be re-run on it: a full physical page number is also a
# valid virtual page of the swa v2p, so the map would silently
# resolve it to an unrelated token's swa slot. Phase 2 of the write
# contract derives the swa loc from them instead.
swa_write_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
if in_capture:
# The runner-built capture batch writes the page-0 sink.
self.full_cg_prefill_swa_out_cache_loc[:num_out].zero_()
else:
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
if translating
else self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
)
)
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(swa_write_loc)
# Captured kernels read the full bucket. Route its inactive tail to
# SWA's zero dummy slot to prevent stale writes into live slots.
self.full_cg_prefill_swa_out_cache_loc[num_out:].zero_()
@@ -826,17 +816,14 @@ class FlashAttentionBackend(AttentionBackend):
def _spec_read_seq_len_delta(
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
) -> int:
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
the translated table's fill must reach ``cache_seqlens``. Takes the
two fields rather than a ForwardBatch: the captured build slices its
batch and has none. Now read by both builds, so eager and replay
widen identically.
"""Columns past ``seq_lens`` this mode's whole-sequence read covers;
the translated table must reach ``cache_seqlens``.
Zero for the prefix-only shapes: normal decode/extend, the topk>1
split (drafts read via the expand metadata), draft-extend whose lens
already include the window, and draft-extend's idle batch. The ragged
verify layout's per-row lens are bounded by the draft window, so its
upper bound rides the same delta."""
split (drafts read via the expand metadata), draft-extend (whose lens
already include the window) and its idle batch. The ragged verify
layout's per-row lens are bounded by the draft window, so the same
delta bounds it."""
if spec_info is None:
return 0
if forward_mode.is_target_verify() and self.topk <= 1:
@@ -1237,27 +1224,20 @@ class FlashAttentionBackend(AttentionBackend):
else None
)
if _unified_read:
delta = self._spec_read_seq_len_delta(
forward_batch.forward_mode, forward_batch.spec_info
# The plan's table covers this forward's reads, a verify's and a
# draft decode's window included; the kernels bound each row by
# `cache_seqlens`.
kv_view = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
)
if delta:
kv_view = self.kv_index_translator.widened_index_table(
forward_batch, seq_len_delta=delta
)
else:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
metadata.page_table = kv_view.ids
if self.use_sliding_window_kv_pool:
metadata.swa_page_table = kv_view.sliding_window_ids
if swa_out_cache_loc is not None:
# The swa write loc was computed from the still-VIRTUAL
# loc at ForwardBatch construction; re-running the
# full->swa map on the physical loc would be garbage.
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
)
)
metadata.swa_page_table = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan,
kind=IdSpaceKind.SLIDING_WINDOW,
rows=forward_batch.batch_size,
).ids
metadata.swa_out_cache_loc = swa_out_cache_loc
elif self.use_sliding_window_kv_pool:
# FA3 requires an int32 page_table.
metadata.swa_page_table = (
@@ -1265,12 +1245,7 @@ class FlashAttentionBackend(AttentionBackend):
metadata.page_table
).to(torch.int32)
)
if swa_out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
swa_out_cache_loc
)
)
metadata.swa_out_cache_loc = swa_out_cache_loc
# Logical-page KV sharding: capture the batch's gather plan and swap the
# page table to scratch rows. During a sharded extend, attention reads
@@ -2864,6 +2839,7 @@ class FlashAttentionBackend(AttentionBackend):
def _set_decode_page_metadata(
self,
plan: KVLocPlan,
metadata,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -2871,9 +2847,8 @@ class FlashAttentionBackend(AttentionBackend):
) -> None:
"""Fill `cache_seqlens_int32`, `cu_seqlens_k` and the page table(s).
Under the unified pool the translator writes the page tables in place,
so the fused kernel is left with the prefix sum alone -- one pass over
the rows instead of a translated build plus a verbatim copy of it.
Under the unified pool the page tables are the plan's, copied in, so
the fused kernel is left with the prefix sum alone.
"""
translated = self.kv_index_translator.reads_are_translated
normal_decode_set_metadata(
@@ -2891,17 +2866,20 @@ class FlashAttentionBackend(AttentionBackend):
skip_page_table=translated,
)
if translated:
# Fill to `cache_seqlens_int32`, which the kernels bound their reads
# by: a draft decode reads `seq_len_delta` past `seq_lens`.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
sliding_window_out=metadata.swa_page_table,
req_pool_indices=req_pool_indices,
seq_lens=metadata.cache_seqlens_int32,
)
# The plan's table reaches `cache_seqlens_int32`, which the kernels
# bound their reads by: a draft decode reads past `seq_lens`.
bs = int(req_pool_indices.numel())
self.kv_index_translator.copy_page_table(plan, out=metadata.page_table[:bs])
if metadata.swa_page_table is not None:
self.kv_index_translator.copy_page_table(
plan,
out=metadata.swa_page_table[:bs],
kind=IdSpaceKind.SLIDING_WINDOW,
)
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -2937,6 +2915,7 @@ class FlashAttentionBackend(AttentionBackend):
# max_seq_len_k left unset -- unread here (scheduler_metadata
# is normal-decode-only).
self._set_decode_page_metadata(
plan,
metadata,
req_pool_indices,
seq_lens,
@@ -3041,6 +3020,7 @@ class FlashAttentionBackend(AttentionBackend):
else self.max_context_len
)
self._set_decode_page_metadata(
plan,
metadata,
req_pool_indices,
seq_lens,
@@ -3092,7 +3072,7 @@ class FlashAttentionBackend(AttentionBackend):
# max_seq_len_k left unset -- unread here (scheduler_metadata is
# normal-decode-only).
self._set_decode_page_metadata(
metadata, req_pool_indices, verify_lens, verify_delta
plan, metadata, req_pool_indices, verify_lens, verify_delta
)
else:
# When topk > 1, we need two specific target verify metadata, and then merge states
@@ -3225,23 +3205,19 @@ class FlashAttentionBackend(AttentionBackend):
metadata.max_seq_len_k + self.page_size - 1
) // self.page_size
if self.kv_index_translator.reads_are_translated:
# The translator writes kernel-facing page ids straight into
# the captured tables; `sliding_window_out` fills the swa twin
# in the same pass.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
sliding_window_out=(
metadata.swa_page_table
if self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
else None
),
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_len_delta=self._spec_read_seq_len_delta(
forward_mode, spec_info
),
rows = int(req_pool_indices.numel())
self.kv_index_translator.copy_page_table(
plan, out=metadata.page_table[:rows]
)
if (
self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
):
self.kv_index_translator.copy_page_table(
plan,
out=metadata.swa_page_table[:rows],
kind=IdSpaceKind.SLIDING_WINDOW,
)
else:
page_indices = self.req_to_token[
req_pool_indices[:, None],
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
@@ -61,6 +62,7 @@ from sglang.srt.utils import (
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -749,6 +751,7 @@ class FlashInferAttnBackend(AttentionBackend):
fixed_split_size=None,
disable_split_kv=self.disable_cuda_graph_kv_split,
req_pool_indices=req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_target_verify():
self.indices_updater_prefill.update(
@@ -761,6 +764,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=spec_info,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_dllm_extend():
self.indices_updater_prefill.update(
@@ -773,6 +777,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=not self.use_paged,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_draft_extend_v2():
self.indices_updater_prefill.update(
@@ -785,6 +790,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=spec_info,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_extend():
# Plain EXTEND under full prefill CUDA graph. plan() runs
@@ -803,6 +809,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
else:
raise ValueError("Invalid forward mode")
@@ -863,8 +870,8 @@ class FlashInferAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
if in_capture:
@@ -955,8 +962,8 @@ class FlashInferAttnBackend(AttentionBackend):
def init_forward_metadata(self, forward_batch: ForwardBatch):
swa_out_cache_loc = None
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
if forward_batch.forward_mode.is_decode_or_idle():
@@ -970,6 +977,7 @@ class FlashInferAttnBackend(AttentionBackend):
fixed_split_size=self.decode_split_tile_size,
disable_split_kv=False,
req_pool_indices=forward_batch.req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = DecodeMetadata(
self.decode_wrappers, swa_out_cache_loc=swa_out_cache_loc
@@ -985,6 +993,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=forward_batch.encoder_lens,
spec_info=forward_batch.spec_info,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrappers_verify,
@@ -1038,6 +1047,7 @@ class FlashInferAttnBackend(AttentionBackend):
cross_attention_custom_mask=forward_batch.cross_attention_custom_mask,
extend_prefix_lens_cpu=forward_batch.extend_prefix_lens_cpu,
custom_kv_indices=self.dq_page_table,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrappers_paged,
@@ -1581,6 +1591,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Keep the signature for type checking. It will be assigned during runtime.
@@ -1597,6 +1608,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
decode_wrappers = decode_wrappers or self.decode_wrappers
@@ -1610,6 +1622,7 @@ class FlashInferIndicesUpdaterDecode:
seq_lens_cpu,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1624,6 +1637,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
assert self.sliding_window_size is not None
@@ -1663,6 +1677,7 @@ class FlashInferIndicesUpdaterDecode:
use_sliding_window_kv_pool=use_sliding_window_kv_pool,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1677,6 +1692,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Cache encoder_lens on CPU to avoid GPU→CPU transfer per call
@@ -1703,6 +1719,7 @@ class FlashInferIndicesUpdaterDecode:
seq_lens_cpu=kv_lens_cpu,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1719,6 +1736,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
@@ -1726,6 +1744,7 @@ class FlashInferIndicesUpdaterDecode:
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
if spec_info is None or getattr(spec_info, "kv_indptr", None) is None:
bs = len(paged_kernel_lens)
kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0)
@@ -1739,14 +1758,14 @@ class FlashInferIndicesUpdaterDecode:
paged_kernel_lens_sum, dtype=torch.int32, device="cuda"
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=use_swa_source,
kind=read_kind,
)
else:
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
@@ -1868,6 +1887,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -1888,6 +1909,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if use_ragged:
assert prefix_lens is not None
@@ -1918,6 +1941,7 @@ class FlashInferIndicesUpdaterPrefill:
multi_item_params=multi_item_params,
seq_lens_cpu=seq_lens_cpu,
custom_kv_indices=custom_kv_indices,
plan=plan,
)
def update_sliding_window(
@@ -1936,6 +1960,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if custom_kv_indices is not None:
raise RuntimeError(
@@ -2031,6 +2057,7 @@ class FlashInferIndicesUpdaterPrefill:
if (wrapper_id == 0 and not use_ragged and spec_info is None)
else -1
),
plan=plan,
)
def _build_swa_prefix_custom_mask(
@@ -2090,6 +2117,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if custom_kv_indices is not None:
raise RuntimeError(
@@ -2125,6 +2154,7 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask=(
cross_attention_custom_mask if wrapper_id == 1 else None
),
plan=plan,
)
def call_begin_forward(
@@ -2148,6 +2178,8 @@ class FlashInferIndicesUpdaterPrefill:
seq_lens_cpu: Optional[torch.Tensor] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
window_left: int = -1,
*,
plan: KVLocPlan,
):
bs = len(seq_lens)
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
@@ -2155,6 +2187,7 @@ class FlashInferIndicesUpdaterPrefill:
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
if spec_info is None:
assert prefix_lens is not None
assert len(seq_lens) == len(req_pool_indices)
@@ -2180,14 +2213,14 @@ class FlashInferIndicesUpdaterPrefill:
dtype=torch.int32,
device=req_pool_indices.device,
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=use_swa_source,
kind=read_kind,
)
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
qo_indptr = qo_indptr[: bs + 1]
@@ -2202,7 +2235,8 @@ class FlashInferIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
plan=plan,
kind=read_kind,
kv_start_idx=kv_start_idx,
)
)
@@ -2213,7 +2247,8 @@ class FlashInferIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
plan=plan,
kind=read_kind,
)
)
@@ -2392,18 +2427,21 @@ class FlashInferMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
@@ -2412,7 +2450,9 @@ class FlashInferMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
FlashInferMlaAttnBackend,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -367,6 +368,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
init_metadata_replay=False,
spec_info=spec_info,
req_pool_indices=req_pool_indices[:bs],
plan=forward_batch.kv_loc_plan,
)
self.decode_cuda_graph_metadata[bs] = decode_wrapper
self.forward_metadata = DecodeMetadata(decode_wrapper)
@@ -391,6 +393,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
raise ValueError(f"Invalid mode: {forward_mode=}")
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -408,6 +411,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
prefill_wrapper.plan = partial(fast_mla_prefill_plan, prefill_wrapper)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -425,6 +429,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
decode_wrapper=self.decode_wrapper,
init_metadata_replay=False,
req_pool_indices=forward_batch.req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = DecodeMetadata(self.decode_wrapper)
elif forward_batch.forward_mode.is_target_verify():
@@ -436,6 +441,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
prefill_wrapper_paged=self.prefill_wrapper_verify,
use_ragged=False,
spec_info=forward_batch.spec_info,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False)
else:
@@ -484,6 +490,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr_cpu=kv_indptr_cpu,
kv_len_arr_cpu=kv_len_arr_cpu,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrapper_paged, use_ragged
@@ -531,6 +538,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -566,6 +574,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
init_metadata_replay=True,
spec_info=spec_info,
req_pool_indices=req_pool_indices[:bs],
plan=plan,
**self.fast_decode_kwargs,
)
elif forward_mode.is_target_verify():
@@ -616,6 +625,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
if use_generic_fast_plan
else None
),
plan=plan,
)
else:
raise ValueError(f"Invalid forward mode: {forward_mode=}")
@@ -859,6 +869,7 @@ class FlashInferMLAIndicesUpdaterDecode:
init_metadata_replay: bool = False,
spec_info: Optional[SpecInput] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
**fast_decode_kwargs,
):
@@ -871,6 +882,7 @@ class FlashInferMLAIndicesUpdaterDecode:
self.kv_indptr,
init_metadata_replay,
spec_info,
plan=plan,
req_pool_indices=req_pool_indices,
**fast_decode_kwargs,
)
@@ -885,6 +897,7 @@ class FlashInferMLAIndicesUpdaterDecode:
init_metadata_replay: bool = False,
spec_info: Optional[SpecInput] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
**fast_decode_kwargs,
):
@@ -904,11 +917,11 @@ class FlashInferMLAIndicesUpdaterDecode:
else fast_decode_kwargs["kv_indices"]
)
translator = self.attn_backend.kv_index_translator
is_translated = translator.fill_packed_read_stream(
is_translated = translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
)
@@ -1006,6 +1019,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
fast_verify_plan_kwargs: Optional[dict] = None,
*,
plan: KVLocPlan,
qo_indptr_cpu: Optional[torch.Tensor] = None,
kv_indptr_cpu: Optional[torch.Tensor] = None,
kv_len_arr_cpu: Optional[torch.Tensor] = None,
@@ -1031,6 +1045,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
spec_info,
attn_dcp_metadata=attn_dcp_metadata,
fast_verify_plan_kwargs=fast_verify_plan_kwargs,
plan=plan,
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr_cpu=kv_indptr_cpu,
kv_len_arr_cpu=kv_len_arr_cpu,
@@ -1052,6 +1067,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
fast_verify_plan_kwargs: Optional[dict] = None,
*,
plan: KVLocPlan,
qo_indptr_cpu: Optional[torch.Tensor] = None,
kv_indptr_cpu: Optional[torch.Tensor] = None,
kv_len_arr_cpu: Optional[torch.Tensor] = None,
@@ -1068,11 +1084,11 @@ class FlashInferMLAIndicesUpdaterPrefill:
dtype=torch.int32,
device=req_pool_indices.device,
)
self.attn_backend.kv_index_translator.fill_packed_read_stream(
self.attn_backend.kv_index_translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
)
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
@@ -1085,6 +1101,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
plan=plan,
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
)
)
@@ -1097,6 +1114,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
plan=plan,
)
)
@@ -1241,25 +1259,30 @@ class FlashInferMLAMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(self.speculative_num_steps),
next_power_of_2(bs),
self.page_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -125,6 +126,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
or forward_mode.is_draft_extend_v2()
):
self._apply_decode_target_verify_metadata(
plan=forward_batch.kv_loc_plan,
bs=forward_batch.batch_size,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
@@ -165,10 +167,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -200,10 +200,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -240,10 +238,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens_k,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -328,6 +324,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
def _apply_decode_target_verify_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -355,10 +352,9 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=self.cuda_graph_kv_indices,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
# The plan's table reaches a verify's draft tail.
self.kv_index_translator.copy_page_table(
plan, out=self.cuda_graph_kv_indices[:bs]
)
else:
create_flashmla_kv_indices_triton[
@@ -251,6 +251,7 @@ def _build_tbo_child_replay_fb_view(
child_seq_lens_cpu = fb_view.seq_lens_cpu[seq_slice]
parent_input_ids = getattr(fb_view, "input_ids", None)
parent_out_cache_loc = getattr(fb_view, "out_cache_loc", None)
plan = getattr(fb_view, "kv_loc_plan", None)
return SimpleNamespace(
batch_size=child_bs,
forward_mode=fb_view.forward_mode,
@@ -270,6 +271,15 @@ def _build_tbo_child_replay_fb_view(
if parent_out_cache_loc is not None
else None
),
# Unified memory refuses two-batch overlap, so the plan's reads stay in
# `req_to_token`, which each child indexes at its own rows; each child
# writes its own tokens of the plan's window.
kv_loc_plan=plan,
kv_loc_cols=(
None
if plan is None
else plan.cols_slice(getattr(fb_view, "kv_loc_cols", None), tok_slice)
),
spec_info=child_spec_info,
max_seq_len_override=fb_view.max_seq_len_override,
)
@@ -404,7 +404,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
if save_kv_cache:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
self._kv_write_loc(forward_batch),
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
@@ -30,6 +30,7 @@ from sglang.srt.layers.dcp import (
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.cuda_graph_config import (
@@ -68,6 +69,7 @@ if _is_cuda:
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -130,9 +132,6 @@ class ForwardMetadata:
swa_attn_logits: Optional[torch.Tensor] = None
# full->SWA translated out_cache_loc (SWA KV-store write target)
swa_out_cache_loc: Optional[torch.Tensor] = None
# PHYSICAL full-attn write target for the unified pool (eager: translated tensor;
# cuda-graph: capture-stable buffer view). None for non-unified pools.
out_cache_loc_full_physical: Optional[torch.Tensor] = None
# Lean decode (persistent-grid partial-result buffers)
lean_Mp: Optional[torch.Tensor] = None
lean_Lp: Optional[torch.Tensor] = None
@@ -503,6 +502,7 @@ class TritonAttnBackend(AttentionBackend):
def _fill_kv_indptr_and_indices(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -512,17 +512,18 @@ class TritonAttnBackend(AttentionBackend):
if kv_indptr is None:
kv_indptr = self.kv_indptr[: bs + 1]
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
self.kv_index_translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
self.kv_index_translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens[:bs],
indptr=kv_indptr,
total_tokens=kv_indices.numel(),
out=kv_indices,
)
return kv_indptr
def _update_decode_kv_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -551,7 +552,7 @@ class TritonAttnBackend(AttentionBackend):
num_kv_splits_lens = dcp_seq_lens.clamp_min(1)
else:
kv_indptr = self._fill_kv_indptr_and_indices(
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
)
num_kv_splits_lens = seq_lens
window_kv_indptr = self.window_kv_indptr
@@ -560,6 +561,7 @@ class TritonAttnBackend(AttentionBackend):
window_kv_indptr, _, window_kv_lens, _ = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
seq_lens,
@@ -577,6 +579,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_target_verify_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
spec_info,
@@ -593,7 +596,7 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
kv_indptr = self._fill_kv_indptr_and_indices(
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
)
window_kv_indptr = self.window_kv_indptr
window_kv_indices = None
@@ -607,6 +610,7 @@ class TritonAttnBackend(AttentionBackend):
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
seq_lens[:bs],
@@ -638,6 +642,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_dllm_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -654,12 +659,13 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
self._fill_kv_indptr_and_indices(
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
)
if self.sliding_window_size is not None and self.sliding_window_size > 0:
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
prefix_lens,
@@ -670,6 +676,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_draft_extend_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
forward_mode: ForwardMode,
@@ -705,7 +712,7 @@ class TritonAttnBackend(AttentionBackend):
extend_seq_lens = torch.zeros(bs, dtype=torch.int32, device=seq_lens.device)
kv_lens = torch.clamp(seq_lens - extend_seq_lens, min=0).to(torch.int32)
kv_indptr = self._fill_kv_indptr_and_indices(
bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
)
return qo_indptr, kv_indptr, num_tokens_per_req
@@ -755,15 +762,13 @@ class TritonAttnBackend(AttentionBackend):
return
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
forward_mode=forward_mode,
spec_info=spec_info,
)
out_cache_loc_full_physical = self._fill_cuda_graph_write_locs(
forward_batch, bs
)
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
forward_batch, in_capture=True
)
@@ -772,25 +777,25 @@ class TritonAttnBackend(AttentionBackend):
forward_mode,
spec_info,
swa_out_cache_loc,
out_cache_loc_full_physical,
)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
forward_mode=forward_mode,
spec_info=spec_info,
)
# Metadata view is reused from capture; just refill the buffers.
self._fill_cuda_graph_write_locs(forward_batch, bs)
# Metadata view is reused from capture; the write loc is the
# runner's slot, so only the swa twin needs refilling.
self._fill_cuda_graph_swa_out_cache_loc(forward_batch)
def _fill_cuda_graph_swa_out_cache_loc(
self, forward_batch: ForwardBatch, in_capture: bool = False
) -> Optional[torch.Tensor]:
"""Refill the SWA write-target buffer from the batch's derived
sliding-window write loc, returning the [:n] view (None for non-SWA /
"""Refill the SWA write-target buffer with the batch's write ids in the
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
multi-step draft) so the captured store reads fresh slots on replay.
"""
if not self.use_sliding_window_kv_pool:
@@ -807,24 +812,12 @@ class TritonAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
return self.cuda_graph_swa_out_cache_loc[:n]
def _fill_cuda_graph_write_locs(
self, forward_batch: ForwardBatch, bs: int
) -> Optional[torch.Tensor]:
"""Runs BEFORE graph.replay(), so it reads the live post-compaction
v2p; no-op for non-unified pools."""
# The buffer exists only for a translating pool; return before naming it.
if not self.kv_index_translator.is_translating:
return None
return self.kv_index_translator.fill_capture_write_loc(
out=self.cuda_graph_out_cache_loc_full_physical,
forward_batch=forward_batch,
width=self.cuda_graph_out_cache_loc_full_physical.numel(),
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Init auxiliary variables for triton attention backend."""
@@ -862,6 +855,7 @@ class TritonAttnBackend(AttentionBackend):
seq_lens_sum, dtype=torch.int64, device=self.device
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.seq_lens,
forward_batch.req_pool_indices,
@@ -875,6 +869,7 @@ class TritonAttnBackend(AttentionBackend):
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.seq_lens,
@@ -961,6 +956,7 @@ class TritonAttnBackend(AttentionBackend):
seq_lens_sum, dtype=torch.int64, device=self.device
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.seq_lens,
forward_batch.req_pool_indices,
@@ -977,6 +973,7 @@ class TritonAttnBackend(AttentionBackend):
) = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.seq_lens,
@@ -1016,6 +1013,7 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.extend_prefix_lens,
forward_batch.req_pool_indices,
@@ -1030,6 +1028,7 @@ class TritonAttnBackend(AttentionBackend):
) = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.extend_prefix_lens,
@@ -1054,8 +1053,8 @@ class TritonAttnBackend(AttentionBackend):
swa_out_cache_loc = None
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
self.forward_metadata = ForwardMetadata(
@@ -1074,11 +1073,6 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets,
swa_attn_logits=swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=(
forward_batch.out_cache_loc
if self.kv_index_translator.is_translating
else None
),
lean_Mp=lean_Mp,
lean_Lp=lean_Lp,
lean_Op=lean_Op,
@@ -1201,22 +1195,12 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
if self.kv_index_translator.is_translating:
# Unified pool full-attention write-target buffer, refilled at replay
# (-> KVWriteLoc.full_loc). Capture-stable, mirrors cuda_graph_swa_out_cache_loc.
self.cuda_graph_out_cache_loc_full_physical = torch.zeros(
(max_num_tokens,),
dtype=torch.int64,
device=self.device,
)
def _build_cuda_graph_forward_metadata(
self,
bs: int,
forward_mode: ForwardMode,
spec_info: Optional[SpecInput],
swa_out_cache_loc: Optional[torch.Tensor] = None,
out_cache_loc_full_physical: Optional[torch.Tensor] = None,
) -> ForwardMetadata:
"""Construct ForwardMetadata from the current cuda-graph buffer state.
@@ -1247,7 +1231,6 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets=None,
swa_attn_logits=self.cuda_graph_swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
lean_Mp=self.cuda_graph_lean_Mp,
lean_Lp=self.cuda_graph_lean_Lp,
lean_Op=self.cuda_graph_lean_Op,
@@ -1279,7 +1262,6 @@ class TritonAttnBackend(AttentionBackend):
),
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_dllm_extend():
return ForwardMetadata(
@@ -1297,7 +1279,6 @@ class TritonAttnBackend(AttentionBackend):
window_num_kv_splits=None,
window_kv_offsets=None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_draft_extend_v2():
return ForwardMetadata(
@@ -1323,13 +1304,13 @@ class TritonAttnBackend(AttentionBackend):
window_num_kv_splits=None,
window_kv_offsets=None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
else:
raise ValueError(f"Invalid forward mode: {forward_mode=} for CUDA Graph.")
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -1344,7 +1325,7 @@ class TritonAttnBackend(AttentionBackend):
if forward_mode.is_decode_or_idle():
assert spec_info is None, "Multi-step cuda graph init is not done here."
_, _, window_kv_lens, num_kv_splits_lens = self._update_decode_kv_buffers(
bs, seq_lens, req_pool_indices
plan, bs, seq_lens, req_pool_indices
)
self.get_num_kv_splits(
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
@@ -1356,13 +1337,13 @@ class TritonAttnBackend(AttentionBackend):
elif forward_mode.is_target_verify():
bs = len(req_pool_indices)
self._update_target_verify_buffers(
bs, seq_lens, spec_info, req_pool_indices
plan, bs, seq_lens, spec_info, req_pool_indices
)
elif forward_mode.is_dllm_extend():
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
self._update_dllm_buffers(plan, bs, seq_lens, req_pool_indices)
elif forward_mode.is_draft_extend_v2():
self._update_draft_extend_buffers(
bs, seq_lens, forward_mode, spec_info, req_pool_indices
plan, bs, seq_lens, forward_mode, spec_info, req_pool_indices
)
else:
raise ValueError(
@@ -1649,7 +1630,6 @@ class TritonAttnBackend(AttentionBackend):
loc_info = KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
)
if layer.k_scale is None:
self._set_kv_buffer(forward_batch, layer, loc_info, k, v)
@@ -2123,8 +2103,6 @@ class TritonAttnBackend(AttentionBackend):
"window-layer extend before the metadata carried a "
"sliding-window write loc"
)
elif self.forward_metadata.out_cache_loc_full_physical is not None:
extend_kv_indices = self.forward_metadata.out_cache_loc_full_physical
# Capture batches may not have a spec_info, so use the attention
# metadata's resolved uniform verify width when extend lengths are absent.
@@ -2234,14 +2212,9 @@ class TritonAttnBackend(AttentionBackend):
k.div_(layer.k_scale)
self.token_to_kv_pool.set_kv_buffer(
layer,
# `full_loc` carries the pre-translated loc under the unified
# pool, refreshed into a capture-stable buffer before replay —
# translating inside set_kv_buffer would be captured and replay
# a stale v2p. None (-> raw loc) for static pools.
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
v,
@@ -2253,7 +2226,6 @@ class TritonAttnBackend(AttentionBackend):
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
v,
@@ -2468,18 +2440,21 @@ class TritonMultiStepDraftBackend:
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
seq_lens_sum = num_seqs * self.max_context_len
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
@@ -2488,7 +2463,9 @@ class TritonMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
if call_fn is None:
@@ -2590,6 +2567,7 @@ class TritonMultiStepDraftBackend:
def update_sliding_window_buffer(
window_kv_indptr,
translator,
plan,
req_pool_indices,
sliding_window_size,
seq_lens,
@@ -2626,16 +2604,18 @@ def update_sliding_window_buffer(
if isinstance(token_to_kv_pool, SWAKVPool)
else None
)
translated = translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
translated = translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=window_kv_lens,
indptr=window_kv_indptr,
total_tokens=window_kv_indices.numel(),
out=window_kv_indices,
kv_start_idx=window_kv_start_idx,
sliding_window=(
translator.reads_are_translated
kind=(
IdSpaceKind.SLIDING_WINDOW
if translator.reads_are_translated
and isinstance(token_to_kv_pool, BaseSWAKVPool)
else IdSpaceKind.FULL
),
token_mapping=token_mapping,
)
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
KVCacheAttentionAccessKind,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -1088,16 +1089,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# fused kernel skips its page-table writes so the graph reads the
# refreshed content through pointers baked at capture).
metadata = self.forward_metadata
# `cache_seqlens_int32` is what the attention kernels bound their
# page-table reads by, and the fused metadata call above wrote it.
# A target verify reads `draft_token_num` further than `seq_lens`
# goes, so filling to `seq_lens` leaves those columns untranslated.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
req_pool_indices=forward_batch.req_pool_indices[:bs],
seq_lens=metadata.cache_seqlens_int32,
sliding_window_out=metadata.swa_page_table,
# The kernels bound their page-table reads by `cache_seqlens_int32`
# (the fused metadata call above wrote it); the plan's table
# reaches that far, a target verify's draft tail included.
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=metadata.page_table[:bs]
)
if metadata.swa_page_table is not None:
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan,
out=metadata.swa_page_table[:bs],
kind=IdSpaceKind.SLIDING_WINDOW,
)
# A capture batch carries no prepared write loc; zeros are the
# page-0 sink.
if (
@@ -1110,8 +1113,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
@@ -1153,7 +1156,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
metadata, req_pool_indices, metadata.cache_seqlens_int32
)
# The fused in-graph kernel also skips ragged batches, so refill the
# SWA write-target buffer here (out_cache_loc -> SWA locs).
# SWA write-target buffer here, from the plan.
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
n = forward_batch.out_cache_loc.shape[0]
self.cuda_graph_swa_out_cache_loc[n:].zero_()
@@ -1161,8 +1164,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
@@ -1266,12 +1269,23 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
metadata.cu_seqlens_q = metadata.cu_seqlens_k
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
kv_view = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
)
if kv_view.is_translated:
# No fill kernel: the kernels take the tensor's own width/stride
# and bound their reads by cache_seqlens.
metadata.page_table = kv_view.ids
metadata.swa_page_table = kv_view.sliding_window_ids
metadata.swa_page_table = (
self.kv_index_translator.read_table(
forward_batch.kv_loc_plan,
kind=IdSpaceKind.SLIDING_WINDOW,
rows=forward_batch.batch_size,
).ids
if self.kv_index_translator.space(IdSpaceKind.SLIDING_WINDOW)
is not None
else None
)
else:
has_swa = self._swa_kv_pool is not None
metadata.page_table = torch.empty(
@@ -1300,10 +1314,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# int64 scatter index (unlike the int32 read page table above).
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
)
metadata.swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
self.forward_metadata = metadata
@@ -79,6 +79,7 @@ if is_flashinfer_available():
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -299,10 +300,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
# Tree-mask scratch is fetched from the target backend only.
self.is_draft_runner = model_runner.is_draft_worker
# [:n] view of a capture-stable buffer on the cuda-graph path; None on
# the eager path, which passes forward_batch.out_cache_loc through.
self._decode_kernel_loc: Optional[torch.Tensor] = None
self.cuda_graph_out_cache_loc_kernel: Optional[torch.Tensor] = None
# Fused KV-scatter + q-concat on the decode dense-loc path (one launch
# instead of set_mla_kv_buffer + concat_mla_absorb_q). Disabled under
# async asserts: the fused path writes the pool directly and would
@@ -411,6 +408,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
def _create_block_kv_indices(
self,
plan: KVLocPlan,
batch_size: int,
max_blocks: int,
req_pool_indices: torch.Tensor,
@@ -430,11 +428,19 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
Returns:
Block KV indices tensor
"""
dcp_enabled = get_parallel().dcp_enabled
translated = self.kv_index_translator.is_translating and not dcp_enabled
# A translated table holds the plan's page ids and reads the sink past
# them: a draft extend pads every row's key length to its widest row,
# and the padded queries (whose outputs are dropped) read that far.
block_kv_indices = torch.full(
(batch_size, max_blocks), -1, dtype=torch.int32, device=device
(batch_size, max_blocks),
0 if translated else -1,
dtype=torch.int32,
device=device,
)
if get_parallel().dcp_enabled:
if dcp_enabled:
self._fill_dcp_block_kv_indices(
block_kv_indices,
req_pool_indices,
@@ -442,12 +448,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
return block_kv_indices
if self.kv_index_translator.is_translating:
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
)
if translated:
self.kv_index_translator.copy_page_table(plan, out=block_kv_indices)
else:
create_flashmla_kv_indices_triton[
(
@@ -480,13 +482,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
self.decode_cuda_graph_kv_indices = torch.full(
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
)
# Unified pool: capture-stable buffer for the physical KV write loc, filled
# out-of-graph in init_forward_metadata_out_graph so the in-graph
# set_mla_kv_buffer captures no translate.
if self.kv_index_translator.is_translating:
self.cuda_graph_out_cache_loc_kernel = torch.zeros(
max_num_tokens, dtype=torch.int64, device=self.device
)
num_tokens_per_req = max_num_tokens // max_bs
if is_float4_e2m1fn_x2(self.data_type):
@@ -603,6 +598,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -642,10 +638,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
# Update block indices for new sequences.
if self.kv_index_translator.is_translating:
self.kv_index_translator.fill_read_table(
out=metadata.block_kv_indices,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
# The plan's table reaches a verify's draft tail.
self.kv_index_translator.copy_page_table(
plan, out=metadata.block_kv_indices[:bs]
)
else:
create_flashmla_kv_indices_triton[
@@ -755,6 +750,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
forward_batch.seq_lens.device,
)
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
@@ -762,56 +758,13 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
forward_mode=forward_mode,
)
# Out-of-graph on capture AND every replay-prep, so the in-graph
# set_mla_kv_buffer captures no translate.
if self.kv_index_translator.is_translating and (
forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()
):
# The captured kernel consumes the whole buffer, so the tail a
# shorter replay leaves must go to slot 0 rather than live pages.
self._decode_kernel_loc = self.kv_index_translator.fill_capture_write_loc(
out=self.cuda_graph_out_cache_loc_kernel,
forward_batch=forward_batch,
width=self.cuda_graph_out_cache_loc_kernel.numel(),
)
else:
self._decode_kernel_loc = None
def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc:
"""The loc an unfused KV scatter must write at: the capture-stable
buffer under a captured unified-pool decode, since the translate
rebinds `out_cache_loc` to a fresh tensor the graph never recorded;
the batch's own loc everywhere else.
"""
if self._decode_kernel_loc is not None:
# Filled by `fill_capture_write_loc`, which translates it.
return KVWriteLoc(self._decode_kernel_loc, physical=True)
return KVWriteLoc.for_batch(forward_batch)
def _resolve_fused_write_loc(
self, forward_batch: ForwardBatch
) -> Optional[torch.Tensor]:
"""Write loc for the fused fp8 KV scatter, or None when this batch is
not covered by it.
Captured decode refills `_decode_kernel_loc` out of the graph, and the
captured kernel must read that buffer. Eager decode on a unified pool
has no such buffer, and the caller falls back to the unfused path.
"""
if self._decode_kernel_loc is not None:
return self._decode_kernel_loc
return (
None
if self.kv_index_translator.is_translating
else forward_batch.out_cache_loc
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Initialize the metadata for a forward pass."""
if self._kv_shard_pool is not None:
@@ -821,7 +774,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
forward_batch,
)
self._decode_kernel_loc = None
# Delegate to parent for non-decode modes.
if (
forward_batch.forward_mode.is_extend()
@@ -912,6 +864,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
max_seqlen_pad = self._calc_padded_blocks(max_seq)
block_kv_indices = self._create_block_kv_indices(
forward_batch.kv_loc_plan,
bs,
max_seqlen_pad,
forward_batch.req_pool_indices,
@@ -1287,7 +1240,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
# Fused: bf16->fp8 quantize + KV scatter + q concat
# in one launch; None when not covered.
@@ -1323,42 +1276,23 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert k is not None and k_rope is not None, (
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
)
if self._decode_kernel_loc is not None:
if merge_query and self._fused_set_kv_concat_q:
# Fused: KV scatter + [q_nope | q_rope] concat in one
# launch; None when the inputs are not covered.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=self._decode_kernel_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
)
else:
# eager (or static pool): out_cache_loc is physical.
if (
merge_query
and self._fused_set_kv_concat_q
and not self.kv_index_translator.is_translating
):
# Static pool only, conservatively.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=forward_batch.out_cache_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# `out_cache_loc` is physical on every pool, eager or captured
# (the runner's slot).
if merge_query and self._fused_set_kv_concat_q:
# Fused: KV scatter + [q_nope | q_rope] concat in one launch;
# None when the inputs are not covered.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=forward_batch.out_cache_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# Prepare query tensor inline (already built when the fused save-KV
# path ran)
@@ -1515,7 +1449,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
# Fused: bf16->fp8 quantize + KV scatter + q concat
# in one launch; None when not covered.
@@ -1551,7 +1485,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
)
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# TODO refactor to avoid code duplication
+2
View File
@@ -604,6 +604,7 @@ class TpModelWorker(BaseTpWorker):
skip_attn_backend_init: Optional[bool] = None, # deprecated
*,
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
return_kv_loc_plan: bool = False,
) -> GenerationBatchResult:
# Get forward batch from schedule batch
if batch is not None:
@@ -649,6 +650,7 @@ class TpModelWorker(BaseTpWorker):
expert_distribution_metrics=out.expert_distribution_metrics,
routed_experts_output=out.routed_experts_output,
indexer_topk_output=out.indexer_topk_output,
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
)
capture_pre_sample_logits(batch, forward_batch, logits_output)
+6
View File
@@ -26,6 +26,7 @@ from sglang.srt.utils.common import async_d2h as _async_d2h
if TYPE_CHECKING:
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.speculative.spec_info import SpecInput
@@ -118,6 +119,11 @@ class GenerationBatchResult:
# until the accepted path comes back over the relay.
spec_verify_out_cache_loc: Optional[torch.Tensor] = None
# The plan the forward wrote its KV through, when the caller asked for it
# (`return_kv_loc_plan`): a speculative worker's own writes into the same
# slots take their ids from it.
kv_loc_plan: Optional[KVLocPlan] = None
# PP+spec: [bs, spec_steps + 1] global node indices of the accepted path.
# Every stage holds the KV for its own layers, so every stage has to compact
# that path into its committed prefix; only the last stage can compute it.
@@ -510,10 +510,9 @@ class KVCacheConfigurator:
token_to_kv_pool = None
# Unified-pool fast path: build req_to_token + token_to_kv pool + allocator
# from one byte buffer, then return. Gated to the target worker by ROLE:
# a compact-window DFLASH draft also passes req_to_token_pool=None, and
# would allocate a SECOND unified byte buffer here. Supports hybrid
# Mamba and hybrid SWA (not DSV4).
# from one byte buffer, then return. Target worker only: a compact-window
# DFLASH draft also passes req_to_token_pool=None. Supports hybrid Mamba
# and hybrid SWA (not DSV4).
if (
get_memory().enable_unified_memory
and req_to_token_pool is None
@@ -581,7 +580,6 @@ class KVCacheConfigurator:
alloc = token_to_kv_pool_allocator
placement = self._fused_draft_from_target_buffer(alloc)
if placement is not None:
# FUSED arm
from sglang.srt.mem_cache.layout.fused_draft import (
draft_swa_layer_ids,
)
@@ -619,7 +617,6 @@ class KVCacheConfigurator:
token_to_kv_pool_allocator=alloc,
unified_memory_pool=None,
)
# PRIVATE arm
draft_virtual_id_space = alloc.draft_virtual_id_space
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
"unified allocator virtual space smaller than the token "
@@ -627,7 +624,7 @@ class KVCacheConfigurator:
f"max_total_num_tokens={sizes.max_total_num_tokens}"
)
# Round UP to page alignment (paged draft backends view the
# pool as (-1, page_size, H, D); the space is not aligned).
# pool as (-1, page_size, H, D); the virtual space is not aligned).
page = max(int(self.pool_page_size or 1), 1)
draft_virtual_id_space = (
(draft_virtual_id_space + page - 1) // page * page
@@ -733,11 +730,8 @@ class KVCacheConfigurator:
)
def _fused_draft_from_target_buffer(self, alloc):
"""The placement this draft binds to, or None for the private-pool
fallback (the draft then owns a raw-virtual-indexed pool of its own).
Target boot declines a placement whenever the fused region cannot
serve the draft (`_fused_draft_decision`), so the private arm is the
rollback lever, not a boot-order bug."""
"""The placement this draft binds to, or None when the target placed
none and the draft builds a private pool over the virtual id space."""
if not (
self.spec_algorithm.is_eagle() or self.spec_algorithm.is_dflash_family()
):
@@ -961,14 +955,10 @@ class KVCacheConfigurator:
def _fused_draft_decision(self):
"""Whether, and where, the draft's layers fuse into the target's
sub-pools. Fusion applies only for: unified memory ON, a unified target
(hybrid-SWA or mamba hybrid, either full-pool kind), an EAGLE-family
or DFLASH/DSPARK algorithm whose draft config was loaded at target
boot. It declines, keeping the draft's private pool, for a non-MHA
draft, a draft backend off the translated MHA rails, DCP, HiCache,
host-pool decode retraction, or an explicit draft KV dtype unlike the
host's; `place_fused_draft` then admits or declines the draft's layer
kinds."""
sub-pools. An empty decision means fusion does not apply (unified
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
loaded at target boot); a declined one says why the draft cannot fuse.
The pool factories decide whether a decline refuses the boot."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
draft_kv_profile,
@@ -998,10 +988,9 @@ class KVCacheConfigurator:
"the fused region holds dense MHA K/V rows"
)
)
# The backend(s) the draft runner will run. A DFLASH-family draft
# worker resolves one as `build_draft_tp_worker` does; an EAGLE draft
# runs the published draft backend, else the target's prefill/decode
# pair. A published draft backend carries model-hook declarations.
# The draft runner's backend, resolved as its worker will: an EAGLE
# draft with no backend of its own runs the target's prefill/decode
# pair. The published draft backend carries model-hook declarations.
if self.spec_algorithm.is_dflash_family():
from sglang.srt.speculative.draft_worker_common import (
resolve_draft_worker_attention_backend,
@@ -1066,10 +1055,9 @@ class KVCacheConfigurator:
else 1
)
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves without the draft model, as the draft runner
# resolves it, so a mismatch is declined here; `auto` follows the
# draft's quant config, which only the draft runner knows, and is
# checked when it binds.
# draft dtype resolves here as the draft runner will; `auto` follows
# the draft's quant config, which only the draft runner knows, so it
# is checked when the draft binds.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
@@ -1098,11 +1086,10 @@ class KVCacheConfigurator:
)
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
pad) for the boot solve's cell model. Single source of truth: assembled
through the same spec the pool factory builds, so the priced entry and
the allocated entry cannot drift. None when the draft does not fuse
into that sub-pool."""
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
pad), built from the same spec as the pool factory's so the priced and
allocated entries agree; None when the draft does not fuse into that
sub-pool."""
placement = self._fused_draft_decision().placement
if placement is None or sub_pool_name != "full":
return None
@@ -1150,14 +1137,11 @@ class KVCacheConfigurator:
def _fused_draft_for_mamba_factory(self):
"""`_fused_draft_for_pool_factory` for a mamba-hybrid host. Its unified
buffer takes the WHOLE profiled KV budget (`unified_total_bytes`), so a
private draft pool -- sized to the full virtual id span -- would be
allocated on top of it, unbudgeted. Until the mamba solve prices that
pool, an EAGLE-family or DFLASH draft that does not fuse is refused here
instead of overcommitting GPU memory. A DSPARK draft that does not fuse
keeps its private pool, unpriced: refusing it would refuse DSPARK on
these hosts whenever its draft declines fusion, as the Kimi-Linear
default (a trtllm_mha draft) does."""
buffer takes the whole profiled KV budget, so a private draft pool
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
does not fuse is refused. A DSPARK draft that does not fuse keeps its
private pool, unpriced, so DSPARK still boots where its draft declines
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
decision = self._fused_draft_decision()
placement = self._fused_draft_for_pool_factory(decision)
if (
@@ -1169,18 +1153,16 @@ class KVCacheConfigurator:
raise ValueError(
"--enable-unified-memory + EAGLE/EAGLE3/DFLASH on a mamba-hybrid "
"target needs the draft's KV fused into the target's pages, but "
f"this draft would keep a private pool ({reason}). A mamba host's "
"unified buffer takes the whole KV budget, so a private draft "
"pool would overcommit GPU memory."
f"this draft cannot fuse ({reason}). A private draft pool would "
"overcommit the KV budget the unified buffer already takes."
)
return placement
def _fused_draft_for_swa_factory(self):
"""`_fused_draft_for_pool_factory` for a hybrid-SWA host. Its boot solve
prices a private EAGLE draft pool at the target's per-token size, not
at the draft's own heads, head_dim and KV dtype, so a draft that does
not fuse is refused here, as on a mamba host, instead of overcommitting
GPU memory."""
the draft's own geometry and KV dtype, so an EAGLE draft that does not
fuse is refused rather than overcommit GPU memory."""
decision = self._fused_draft_decision()
placement = self._fused_draft_for_pool_factory(decision)
if (
@@ -1192,15 +1174,14 @@ class KVCacheConfigurator:
raise ValueError(
"--enable-unified-memory + EAGLE/EAGLE3 on a hybrid-SWA target "
"needs the draft's KV fused into the target's pages, but this "
f"draft would keep a private pool ({reason}), which the unified "
"pool's sizing does not price."
f"draft cannot fuse ({reason}), and the unified pool's sizing "
"does not price a private draft pool."
)
return placement
def _fused_draft_for_pool_factory(self, decision):
"""The placement a pool factory is handed, from ONE resolved decision,
so the boot log reports exactly that placement: a declined fusion and
an engaged one otherwise look identical from outside."""
"""The placement a pool factory is handed. It logs the decision it
returns, so the boot log shows whether fusion engaged or why not."""
if decision.placement is None:
if decision.declined is not None:
logger.warning("fused draft KV disabled: %s", decision.declined)
+418 -397
View File
@@ -23,18 +23,29 @@ A KV slot can be named in two id spaces:
The two coincide on a plain pool, so nothing here does any work there.
Backends get a `KVIndexTable`, which answers "what do I gather from, and
which row is mine?":
A pool can hold several sub-pools (full attention, sliding window), each with
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
once, by the readers below and nowhere else; a backend never translates. The
plan's read table exists only once a table reader asks for it (`read_table`,
`copy_page_table`), built into that reader's own capture-stable table when it
is the first; later readers take it, earlier ones translate in their own
gather. The readers answer "what do I gather from, and which row is mine?"
with a `KVIndexTable`:
ids[row_ids[b], pos]
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
objects - no copy, no kernel)
unified : ids = a built array of physical ids,
row_ids = arange(batch_size)
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
objects - no copy, no kernel)
shared table : ids = the plan's array of physical page ids,
row_ids = arange(batch_size)
no table : ids = req_to_token, row_ids = req_pool_indices, and
v2p = the sub-pool's page table, applied by the gather
Backends call their own copy a *page table* (fa3) or a *block table*
(trtllm); here it is the **index table**.
(trtllm); here it is the **index table**. `read_source` serves a gather
kernel, `pack_read_stream` a CSR stream, and `read_table` / `copy_page_table`
a page-table kernel.
Converting only ever rewrites the page number and keeps the in-page offset, so
one page-granular table serves both kinds of consumer: a block-table backend
@@ -42,16 +53,15 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
token_id = entry * entry_page_size + pos % entry_page_size
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
ids at ForwardBatch construction - the earliest consumer can snapshot it
right after. The sliding-window write loc is derived at the same moment as
read table, into the same index table.
The one translation outside a plan is DCP's read production
(`translate_dcp_read_ids`): which ids a rank reads is only known where the
DCP index kernels select them.
"""
from __future__ import annotations
import weakref
from typing import Optional, Tuple
import functools
from typing import Dict, Optional
import msgspec
import torch
@@ -63,6 +73,7 @@ from sglang.kernels.ops.kvcache.kv_read_table import (
build_kv_read_table,
build_kv_read_table_packed,
)
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
@@ -71,30 +82,36 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
)
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.kv_loc_plan import (
IdSpace,
IdSpaceKind,
KVLocPlan,
pad_with_sink,
window_read_extent,
)
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_npu
class KVReadTables(msgspec.Struct, frozen=True):
"""One capture-stable destination, however many id spaces the pool has.
A backend holds this and hands it back to `build_index_table(into=...)`;
it never has to know whether there is a sliding-window space behind it.
"""
full: torch.Tensor
sliding_window: Optional[torch.Tensor]
_is_npu = is_npu()
class KVIndexTable(msgspec.Struct, frozen=True):
"""Collection of what one batch gathers from."""
"""What one batch gathers from, and how its entries become the ids a
kernel reads: lane ``b``'s entry ``j`` is ``ids[row_ids[b] * row_stride +
j // entry_page_size]``, expanded to tokens when an entry is a page, then
mapped through ``v2p`` (page-granular, at the translator's page size) when
one is given. A gather kernel executes exactly this; it never decides
whether or how to translate."""
ids: torch.Tensor # 2-D array of KV ids to gather from
row_ids: torch.Tensor # which row belongs to batch lane b
row_stride: int # stride between rows of `ids`, in elements
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
is_translated: bool # entries are already physical ids
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
is_translated: bool # the ids a gather yields are physical
# Virtual rows a gather translates on the way out (no shared table this
# iteration); None when the entries are already what the kernel reads.
v2p: Optional[torch.Tensor] = None
class KVIndexTranslator:
@@ -108,6 +125,7 @@ class KVIndexTranslator:
token_to_kv_pool,
page_size: int,
device: str,
is_draft_worker: bool = False,
):
self.req_to_token = req_to_token
self.page_size = page_size
@@ -125,55 +143,104 @@ class KVIndexTranslator:
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
)
self.is_translating = is_unified_target or is_fused_draft
self._spaces: Dict[IdSpaceKind, IdSpace] = {}
if self.is_translating:
alloc = token_to_kv_pool_allocator
self._capture_page_size = alloc.page_size
self._full_v2p_table = alloc.full_v2p_page_table
self._full_p2v_table = alloc.full_p2v_page_table
self._translate_full = alloc.translate_kv_loc
# The WRITE loc is the one id that arrives DCP-WIDENED: read indices
# are collapsed by the DCP index kernels, `out_cache_loc` still
# carries the owner rule in `loc % dcp_size`. Identity with the read
# translate when dcp_size == 1.
self._translate_write_full = alloc.translate_write_loc
# DCP read ids stay WIDENED to the consumer: selecting this rank's
# share changes the length, so only the production site can do it.
self.defer_read_translate = get_parallel().attn_dcp_size > 1
# Keyed by the allocator, which owns the tables: a fused draft's
# runner shares the target's full sub-pool, and so its key.
self._spaces[IdSpaceKind.FULL] = IdSpace(
key=(IdSpaceKind.FULL, id(alloc)),
# The WRITE loc is the one id that arrives DCP-WIDENED: read
# indices are collapsed by the DCP index kernels, `out_cache_loc`
# still carries the owner rule in `loc % dcp_size`.
write=alloc.translate_write_loc,
read_v2p=None if self.defer_read_translate else self._full_v2p_table,
)
routes_window_layers = is_unified_target or isinstance(
token_to_kv_pool, BaseSWAKVPool
)
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
self._swa_v2p_table = alloc.swa_v2p_page_table
self._swa_write_loc_from_full = self._swa_write_loc_unified
else:
self._swa_v2p_table = None
self._swa_write_loc_from_full = None
# Straight from the virtual ids through the sliding-window
# sub-pool's own table.
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
key=(IdSpaceKind.SLIDING_WINDOW, id(alloc)),
write=alloc.translate_loc_from_full_to_swa,
read_v2p=(
None if self.defer_read_translate else alloc.swa_v2p_page_table
),
)
else:
self._capture_page_size = page_size
self._full_v2p_table = None
self._full_p2v_table = None
self._translate_full = None
self._translate_write_full = None
self.defer_read_translate = False
self._swa_v2p_table = None
parallel = get_parallel()
if _is_npu and parallel.dcp_enabled and not is_draft_worker:
# An NPU DCP target writes its rank's share of the window in
# rank-local slots (-1: another rank's); the scheduler,
# `req_to_token` and the replicated draft pool keep the
# allocator-global ones the window is planned in.
localize = functools.partial(
localize_dcp_indices,
dcp_size=parallel.dcp_size,
dcp_rank=parallel.dcp_rank,
interleave_size=page_size,
)
self._spaces[IdSpaceKind.FULL] = IdSpace(
key=(IdSpaceKind.FULL, "dcp-rank-local"), write=localize
)
else:
localize = None
self._spaces[IdSpaceKind.FULL] = IdSpace(key=(IdSpaceKind.FULL, None))
# `translate_loc_from_full_to_swa` is abstract on `BaseSWAKVPool`,
# which is also what the backends' `_resolve_swa_kv_pool` keys on.
self._swa_write_loc_from_full = (
token_to_kv_pool.translate_loc_from_full_to_swa
if isinstance(token_to_kv_pool, BaseSWAKVPool)
and (
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
or token_to_kv_pool.request_window is None
# Reads stay in `req_to_token`; the backends map them themselves.
if isinstance(token_to_kv_pool, BaseSWAKVPool) and (
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
or token_to_kv_pool.request_window is None
):
to_swa = token_to_kv_pool.translate_loc_from_full_to_swa
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
key=(IdSpaceKind.SLIDING_WINDOW, id(token_to_kv_pool)),
write=(
to_swa
if localize is None
else lambda ids: to_swa(localize(ids))
),
)
else None
)
self._rows: Optional[torch.Tensor] = (
torch.arange(req_to_token.shape[0], dtype=torch.int64, device=device)
if self.is_translating
else None
)
self._index_table_memo: Optional[Tuple[weakref.ref, KVIndexTable]] = None
def space(self, kind: IdSpaceKind) -> Optional[IdSpace]:
"""How this runner's ``kind`` sub-pool names the iteration's slots, or
None when its pool has no such sub-pool."""
return self._spaces.get(kind)
def write_ids(
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
) -> Optional[torch.Tensor]:
"""The ids ``forward_batch`` writes in this runner's ``kind`` sub-pool,
taken from its plan and padded with the sink to its `out_cache_loc`'s
length (a padded batch, a captured graph's buffer). None when there is
no write loc or no such sub-pool."""
plan = getattr(forward_batch, "kv_loc_plan", None)
out_cache_loc = getattr(forward_batch, "out_cache_loc", None)
if plan is None or out_cache_loc is None:
return None
ids = plan.write_ids(
self, kind=kind, cols=getattr(forward_batch, "kv_loc_cols", None)
)
return pad_with_sink(ids, out_cache_loc.shape[0])
def capture_token_capacity(self, max_token_pool_size: int) -> int:
"""Host capture rows are indexed by request-token IDs, not kernel IDs.
@@ -185,240 +252,305 @@ class KVIndexTranslator:
return self._full_v2p_table.numel() * self._capture_page_size
return max_token_pool_size + self.page_size
def full_flat_v2p(self) -> Optional[torch.Tensor]:
"""The full-side v2p page table for a kernel that translates flat ids
itself, or ``None`` when this runner does not translate (pass-through
and static pools)."""
if not self.is_translating:
return None
return self._full_v2p_table
# -- the iteration's plan --------------------------------------------------
# -- per-batch view --------------------------------------------------------
def plan(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
) -> KVLocPlan:
"""This iteration's ids, translated once. ``write_virtual`` is the
iteration's write window (``[batch, window]`` flattened when its
forwards write columns of it); ``read_extent`` is how far past
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
return KVLocPlan(
source=self,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=write_virtual,
read_extent=read_extent,
write_slots=write_slots,
)
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
"""The plan of a forward that is its own iteration, from its own
fields: its ``out_cache_loc`` is the write window (virtual, or with
``runner_slots`` a runner's own buffer), and its reads reach
`window_read_extent` past its lengths."""
write_ids = forward_batch.out_cache_loc
seq_lens = getattr(forward_batch, "seq_lens", None)
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
encoder_lens = getattr(forward_batch, "encoder_lens", None)
if (
encoder_lens is not None
and seq_lens is not None
and self.reads_are_translated
):
# An encoder-decoder row holds its encoder tokens ahead of the
# decoder's, and its self-attention reads past `seq_lens`.
seq_lens = seq_lens + encoder_lens
encoder_lens_cpu = getattr(forward_batch, "encoder_lens_cpu", None)
seq_lens_cpu = (
None
if seq_lens_cpu is None or encoder_lens_cpu is None
else seq_lens_cpu + torch.tensor(encoder_lens_cpu).to(seq_lens_cpu)
)
return self.plan(
req_pool_indices=getattr(forward_batch, "req_pool_indices", None),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=None if runner_slots else write_ids,
write_slots=write_ids if runner_slots else None,
read_extent=window_read_extent(
getattr(forward_batch, "forward_mode", None),
getattr(forward_batch, "spec_info", None),
write_ids,
getattr(forward_batch, "batch_size", 0),
),
)
def bind_runner_slots(self, forward_batch) -> None:
"""Bind a batch a runner builds over its own buffers, to capture a
graph or to warm up with. Its write ids are the runner's buffer, used
as it is; its reads plan over the runner's own rows and lengths."""
self.own_plan(forward_batch, runner_slots=True).bind(forward_batch, self)
def build_iteration_table(
self,
plan: KVLocPlan,
space: IdSpace,
*,
rows: Optional[int],
previous: Optional[KVIndexTable],
into: Optional[torch.Tensor] = None,
) -> KVIndexTable:
"""Build ``plan``'s page table in ``space`` for `KVLocPlan.read_table`:
the passthrough where reads stay virtual, else one build over ``[0,
seq_lens + read_extent)``. Rows past the plan's batch read the sink; a
request wider than ``previous`` copies its rows rather than translating
again. ``into``, the asking reader's capture-stable table, is built in
place when wide enough and then serves as the plan's table."""
if space.read_v2p is None:
return self._passthrough_table(plan.req_pool_indices)
bs = int(plan.req_pool_indices.numel())
rows = max(bs, rows or 0)
if previous is not None:
out = torch.zeros(
(rows, previous.ids.shape[1]), dtype=torch.int32, device=self.device
)
out[: previous.ids.shape[0]].copy_(previous.ids)
else:
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
slc = plan.seq_lens_cpu
if slc is not None and slc.numel() > 0:
max_seq = int(slc.max()) + plan.read_extent
width = min(max(-(-max_seq // self.page_size), 1), row_pages)
elif into is not None:
# No host lengths: a row's speculative headroom outruns the
# captured table, which is as wide as its graph reads.
width = min(into.shape[1], row_pages)
else:
width = row_pages
# `zero_tail` writes the sink past each live prefix, so neither a
# reused `into` nor an `empty` table keeps stale columns.
if into is not None and into.shape[0] >= rows and into.shape[1] >= width:
out = into[:rows, :width]
else:
out = torch.empty((rows, width), dtype=torch.int32, device=self.device)
if rows > bs:
out[bs:].zero_()
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=plan.req_pool_indices,
seq_lens=plan.seq_lens,
v2p=space.read_v2p,
page_size=self.page_size,
max_pages=width,
out=out,
zero_tail=True,
seq_len_delta=plan.read_extent,
)
return KVIndexTable(
ids=out,
row_ids=self._rows[:rows],
row_stride=out.stride(0),
entry_page_size=self.page_size,
is_translated=True,
)
# -- readers ---------------------------------------------------------------
def _reads_translated(self, kind: IdSpaceKind) -> bool:
space = self._spaces.get(kind)
return space is not None and space.read_v2p is not None
def read_source(
self,
plan: KVLocPlan,
*,
req_pool_indices: torch.Tensor,
bs: int,
kind: IdSpaceKind = IdSpaceKind.FULL,
) -> KVIndexTable:
"""The row source a gather kernel reads lane ``b``'s ``kind`` ids from:
the plan's table once a table reader has had it built, else the virtual
`req_to_token` rows with ``v2p`` for the kernel's gather to apply (no
table is built for it). The plain `req_to_token` passthrough where
reads stay virtual (a static pool, or DCP)."""
if not self._reads_translated(kind):
return self._passthrough_table(req_pool_indices)
if plan.has_read_table(kind):
return self._plan_table(plan, kind=kind, rows=bs)
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=True,
v2p=self.space(kind).read_v2p,
)
def read_table(
self,
plan: KVLocPlan,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
rows: Optional[int] = None,
) -> KVIndexTable:
"""The plan's ``kind`` page table, exactly ``rows`` rows when given
(lanes past the plan's batch read the sink); the ``req_to_token``
passthrough where reads stay virtual (a static or private pool, or DCP,
where the producing kernel selects this rank's share)."""
if self._reads_translated(kind):
return self._plan_table(plan, kind=kind, rows=rows)
return self._passthrough_table(plan.req_pool_indices)
def _plan_table(
self,
plan: KVLocPlan,
*,
kind: IdSpaceKind,
rows: Optional[int] = None,
into: Optional[torch.Tensor] = None,
):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
"req_to_token rows"
)
return plan.read_table(kind=kind, rows=rows, into=into)
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=False,
)
def pack_read_stream(
self,
plan: KVLocPlan,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
indptr: torch.Tensor,
out: torch.Tensor,
kv_start_idx: Optional[torch.Tensor] = None,
kind: IdSpaceKind = IdSpaceKind.FULL,
token_mapping: Optional[torch.Tensor] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
over; return whether they are physical. Packed from the plan's table
once a table reader has had it built, else gathered and translated
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
(a static pool, or DCP) for the caller to finish; a static SWA pool can
fuse its full->swa table into the gather as ``token_mapping``."""
bs = int(seq_lens.numel())
if self._reads_translated(kind) and not plan.has_read_table(kind):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
"req_to_token rows"
)
# `seq_lens` sizes the batch: a caller may hold a wider
# req_pool_indices (a padded graph buffer), whose extra lanes have
# no length to bound.
assert req_pool_indices.numel() >= bs, (
f"pack_read_stream: {bs} stream lanes, "
f"{req_pool_indices.numel()} request rows"
)
build_kv_read_table_packed(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
v2p=self.space(kind).read_v2p,
indptr=indptr,
page_size=self.page_size,
max_tokens=out.numel(),
out=out,
kv_start_idx=kv_start_idx,
)
return True
# The plan's table or a passthrough: the entries are the stream's ids.
src = self.read_source(
plan, req_pool_indices=req_pool_indices, bs=bs, kind=kind
)
assert src.v2p is None, "pack_read_stream: a table-less translating read"
assert token_mapping is None or not src.is_translated
create_flashinfer_kv_indices_triton[(bs,)](
src.ids,
src.row_ids[:bs],
seq_lens,
indptr,
kv_start_idx,
out,
src.row_stride,
ENTRY_PAGE_SIZE=src.entry_page_size,
token_mapping=token_mapping,
)
return src.is_translated or token_mapping is not None
def copy_page_table(
self,
plan: KVLocPlan,
*,
out: torch.Tensor,
kind: IdSpaceKind = IdSpaceKind.FULL,
) -> None:
"""Fill ``out``, a captured graph's page table (padded lanes read the
sink), with the plan's ``kind`` table. The plan's first table reader
builds straight into ``out``, which then serves as the plan's table; a
later one copies it. Columns past the plan's width keep stale values,
which kernels never read past their own lengths."""
assert self._reads_translated(kind), (
"copy_page_table: reads stay virtual here (a non-unified pool, or "
"DCP, where the caller selects this rank's share itself)"
)
rows = out.shape[0]
first = not plan.has_read_table(kind)
table = self._plan_table(
plan, kind=kind, rows=rows, into=out if first else None
)
if table.ids.data_ptr() == out.data_ptr():
return # built in place, or this very buffer is the plan's table
width = min(out.shape[1], table.ids.shape[1])
out[:, :width].copy_(table.ids[:rows, :width])
# -- dispositions ----------------------------------------------------------
@property
def reads_are_translated(self) -> bool:
"""Whether a read this translator fills comes out physical. False
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
``translate_dcp_read_ids`` to finish."""
return self.is_translating and not self.defer_read_translate
def fill_packed_read_stream(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
indptr: torch.Tensor,
total_tokens: int,
out: torch.Tensor,
kv_start_idx: Optional[torch.Tensor] = None,
sliding_window: bool = False,
token_mapping: Optional[torch.Tensor] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
report whether they came out translated.
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
Unified: one fused gather-and-translate, so no caller needs a
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
resident token, a length the pool bounds.
``sliding_window`` selects the swa sub-pool's own id space, built from
VIRTUAL ids and never chained through full-physical. A ``False`` return
means the ids are still VIRTUAL: the DCP path defers translation to
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
table as ``token_mapping`` to fuse that translation into the gather.
"""
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
# (the padded graph buffer), and the extra lanes have no length to bound.
bs = int(seq_lens.numel())
assert req_pool_indices.numel() >= bs, (
f"fill_packed_read_stream: {req_pool_indices.numel()} req rows for "
f"{bs} lengths"
)
req_pool_indices = req_pool_indices[:bs]
if not self.reads_are_translated:
create_flashinfer_kv_indices_triton[(bs,)](
self.req_to_token,
req_pool_indices,
seq_lens,
indptr,
kv_start_idx,
out,
self.req_to_token.stride(0),
ENTRY_PAGE_SIZE=1,
token_mapping=token_mapping,
)
return token_mapping is not None
if sliding_window:
assert self._swa_v2p_table is not None, (
"fill_packed_read_stream: sliding_window on a pool with no swa sub-pool"
)
build_kv_read_table_packed(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._swa_v2p_table if sliding_window else self._full_v2p_table,
indptr=indptr,
page_size=self.page_size,
max_tokens=total_tokens,
out=out,
kv_start_idx=kv_start_idx,
)
return True
def build_index_table(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
max_pages: Optional[int] = None,
into: Optional[KVReadTables] = None,
seq_len_delta: int = 0,
) -> KVIndexTable:
"""The one per-batch entry point.
Non-unified: the raw ``(req_to_token, req_pool_indices)`` passthrough,
no tensor ops and no copies. Unified: fills each row's live prefix and
returns the table WHOLE, so a caller needing a stable pointer (a
captured graph bakes it) passes its own tables in ``into``;
``into=None`` allocates of width ``max_pages`` instead.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). An eager caller's
``max_pages`` must already cover the delta.
"""
if not self.is_translating or self.defer_read_translate:
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=False,
sliding_window_ids=None,
)
bs = int(req_pool_indices.numel())
if into is not None:
out_full = into.full
out_swa = into.sliding_window
# A caller-owned table may be padded wider than req_to_token's span
# (trtllm_mla / flashmla pad to a page-count bound); the columns
# past it have no source to read, so stop there.
width = min(
out_full.shape[1] if max_pages is None else max_pages,
-(-self.req_to_token.shape[1] // self.page_size),
)
else:
assert max_pages is not None, (
"KVIndexTranslator.build_index_table: allocating needs max_pages "
"(from the batch's seq_lens_cpu max)"
)
width = max_pages
out_full = torch.zeros((bs, width), dtype=torch.int32, device=self.device)
out_swa = (
torch.zeros((bs, width), dtype=torch.int32, device=self.device)
if self._swa_v2p_table is not None
else None
)
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._full_v2p_table,
page_size=self.page_size,
max_pages=width,
out=out_full,
seq_len_delta=seq_len_delta,
)
if out_swa is not None:
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._swa_v2p_table,
page_size=self.page_size,
max_pages=width,
out=out_swa,
seq_len_delta=seq_len_delta,
)
return KVIndexTable(
ids=out_full,
row_ids=self._rows[:bs],
row_stride=out_full.stride(0),
entry_page_size=self.page_size,
is_translated=True,
sliding_window_ids=out_swa,
)
def fill_read_table(
self,
*,
out: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
sliding_window_out: Optional[torch.Tensor] = None,
seq_len_delta: int = 0,
) -> None:
"""`build_index_table(into=...)` for a caller that owns a bare block
table rather than a KVReadTables: the page-table consumers read that
table directly, its rows already being the index table's rows.
`sliding_window_out` fills the swa twin in the same pass, for a hybrid
model whose kernels take two block tables.
"""
# `reads_are_translated`, not `is_translating`: under DCP the builder
# returns the passthrough view and writes nothing, so `is_translating`
# would let a caller keep a stale table and never hear about it.
assert self.reads_are_translated, (
"KVIndexTranslator.fill_read_table cannot fill a page table when "
"reads stay virtual (a non-unified pool, or DCP, where the caller "
"must select this rank's share itself)"
)
assert sliding_window_out is None or self._swa_v2p_table is not None, (
"KVIndexTranslator.fill_read_table: asked for a sliding-window "
"table on a pool with no swa sub-pool"
)
self.build_index_table(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
into=KVReadTables(full=out, sliding_window=sliding_window_out),
seq_len_delta=seq_len_delta,
)
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
"""Eager per-batch view, memoized in one slot keyed by batch identity
so multi-consumer metadata builds share a build. The next batch
replaces the slot; consumers only read during their own build. The
captured path does not memoize -- it refreshes its buffers per
replay."""
memo = self._index_table_memo
if memo is not None and memo[0]() is forward_batch:
return memo[1]
max_pages = None
if self.is_translating:
# `seq_lens_cpu` is a non-None but STALE slice on a gpu_only
# batch; `seq_lens_sum` is the signal that it is live. A stale max
# under-sizes the table and the tail then reads as the sink.
slc = forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is not None
and slc is not None
and slc.numel() > 0
):
max_seq = int(slc.max())
else:
max_seq = self.req_to_token.shape[1]
max_pages = max(-(-max_seq // self.page_size), 1)
view = self.build_index_table(
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
max_pages=max_pages,
)
self._index_table_memo = (weakref.ref(forward_batch), view)
return view
return self._reads_translated(IdSpaceKind.FULL)
@property
def full_v2p_table(self) -> Optional[torch.Tensor]:
@@ -426,43 +558,11 @@ class KVIndexTranslator:
pool needs no translation.
For the DCP page-table builders, whose gather is over a rank's cyclic
slice rather than a row prefix, so `build_index_table` cannot serve
them.
slice rather than a row prefix, so a plan's row-prefix table cannot
serve them.
"""
return self._full_v2p_table
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
"""Whole-sequence-verify view: every row's live prefix widened by
`seq_len_delta` columns (the drafts are read back from the pool).
Not memoized -- verify is one consumer, and the batch's memoized
prefix table stays valid for the others.
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
carries `seq_lens_cpu` already expanded by the window and keeps the
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
lens would count the window twice. The width never exceeds the
`req_to_token` row it is read from."""
max_pages = None
if self.is_translating:
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
slc = live if live is not None else forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is not None
and slc is not None
and slc.numel() > 0
):
max_seq = int(slc.max()) + seq_len_delta
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
else:
max_pages = row_pages
return self.build_index_table(
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
max_pages=max_pages,
seq_len_delta=seq_len_delta,
)
def bind_and_verify_backends(self, backends) -> None:
"""Boot: make every reachable backend carry THIS translator.
@@ -481,84 +581,13 @@ class KVIndexTranslator:
"copy, not build its own."
)
# -- write loc (phase 1; phase 2 lives in build_index_table) ----------------
def bind_own_plan(self, forward_batch) -> None:
"""Bind a batch built outside `ForwardBatch.init_new` that is its own
iteration to a plan of its own fields (`own_plan`). A forward that
shares its iteration's window takes that plan instead."""
self.own_plan(forward_batch).bind(forward_batch, self)
def rebind_write_loc(self, forward_batch) -> None:
"""Phase 1 of the WRITE contract: translate the batch's write loc to
FULL-side physical ids exactly once, at ForwardBatch construction, and
mark it physical. On non-unified pools the allocation is already
physical, so only the mark is set.
REBIND, never mutate: the translate returns a FRESH tensor, so the
ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept /
in-flight machinery that reads it. The pre-translate tensor stays on
the batch for `fill_capture_write_loc`.
"""
self._index_table_memo = None
if forward_batch.out_cache_loc is None:
return
if self.is_translating:
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
forward_batch.out_cache_loc = self._translate_write_full(
forward_batch.out_cache_loc
)
forward_batch.out_cache_loc_is_physical = True
def fill_capture_write_loc(
self,
*,
out: torch.Tensor,
forward_batch,
width: Optional[int] = None,
) -> Optional[torch.Tensor]:
"""Translate this batch's WRITE loc straight into ``out``, a backend's
capture-stable buffer, and return the live ``[:n]`` view. One launch
fills the live prefix and clears the tail a shorter replay leaves;
None when this pool needs no translation.
Must run at metadata-init time: `out` is reused every step, so filling
it sooner would race a still-pending previous step under overlap
scheduling.
"""
if not self.is_translating:
return None
virtual = forward_batch.out_cache_loc_virtual
if virtual is None:
loc = forward_batch.out_cache_loc
if loc is None:
return None
# The runner builds the capture batch outside `init_new`, so no
# rebind marked its virtual source; bake it holding sink ids.
width = int(loc.numel()) if width is None else int(width)
out[:width].zero_()
return out[: int(loc.numel())]
n = int(virtual.numel())
width = n if width is None else int(width)
buf = out[:width]
self._translate_write_full(virtual, out=buf, out_width=width)
return buf[:n]
def sliding_window_write_loc_for(
self, out_cache_loc: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
"""This batch's sliding-window write loc, or None when there is no loc
this forward or the pool has no sliding-window id space."""
if out_cache_loc is None or self._swa_write_loc_from_full is None:
return None
return self._swa_write_loc_from_full(out_cache_loc)
def _swa_write_loc_unified(self, full_loc: torch.Tensor) -> torch.Tensor:
"""Sliding-window write loc, derived pointwise from FULL-side physical
ids (phase 2 of the write contract).
"""
ps = self.page_size
offset = full_loc % ps # == virtual_token % page_size
# An unmapped physical page reads back as -1; clamp it rather than let
# the gather wrap onto the v2p table's last element.
virt_page = self._full_p2v_table[full_loc // ps].clamp_(min=0)
return (self._swa_v2p_table[virt_page] * ps + offset).clamp_(min=0)
# -- token-level translate surface (the mixin / local-attn consumers) ------
# -- DCP read production ---------------------------------------------------
@property
def needs_read_translate(self) -> bool:
@@ -575,14 +604,6 @@ class KVIndexTranslator:
dcp_size = get_parallel().attn_dcp_size
if dcp_size > 1:
widened_ids = widened_ids // dcp_size
return self.translate_full_attn_ids(widened_ids)
def translate_full_attn_ids(
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""Virtual token ids -> physical full-attention ids (the identity
when no translation is needed, so callers never branch)."""
if not self.is_translating:
assert out is None, "passthrough translate takes no out="
return kv_indices
return self._translate_full(kv_indices, out=out)
return widened_ids
return self._translate_full(widened_ids)
+348
View File
@@ -0,0 +1,348 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""One scheduler iteration's KV slot ids, per sub-pool.
Under the unified pool a KV slot has a virtual id (what the scheduler allocates
and `req_to_token` stores) and, in each sub-pool, a physical id (what that
sub-pool's kernels index). Every forward of one iteration -- the target's, and
a fused draft's, which lives in the target's pages -- writes the same slots and
reads the same rows, and compaction does not move a page while the iteration's
forwards are in flight. So:
* The write window is translated once per sub-pool, by the plan: the
full-attention ids when the plan is built, another sub-pool's on first
use (`write_ids`). Every writer binds them.
* A read is translated exactly once, only by the translator's read
primitives (`KVIndexTranslator`); a consumer never translates.
* The read table (`read_table`) exists only once a table reader asks for
it, built into that reader's own capture-stable table when it is the
first. Later readers take it; earlier ones translate in their own gather.
* A sub-pool is named by its `IdSpace`; one that indexes virtual ids as they
are (a static pool's, a private draft pool's) translates nothing.
One plan per iteration: `ForwardBatch.init_new` builds it for a lone forward, a
speculative worker as soon as the iteration's slots are allocated (shared by
its draft and target forwards and direct KV writers), and a graph runner for a
batch over its own buffers (`write_slots`).
"""
from __future__ import annotations
import enum
from typing import TYPE_CHECKING, Callable, Dict, Hashable, Optional, Union
import msgspec
import torch
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_index_translator import (
KVIndexTable,
KVIndexTranslator,
)
class IdSpaceKind(enum.Enum):
"""A sub-pool of a runner's KV pool, as the consumers that index it name
it."""
# Full-attention layers: the pool's own token ids.
FULL = enum.auto()
# The sliding-window layers of a hybrid pool: the same token ids through
# the sliding-window sub-pool's table.
SLIDING_WINDOW = enum.auto()
class IdSpace(msgspec.Struct, frozen=True):
"""How one sub-pool names the iteration's slots.
``write`` maps the iteration's virtual token ids to this sub-pool's ids
(None: they index it as they are). ``read_v2p`` is the page table its reads
go through (None: reads stay in `req_to_token`, for the consumer to
finish). Runners whose pools share a sub-pool share its ``key``, and with
it a plan's ids for it."""
key: Hashable
write: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
read_v2p: Optional[torch.Tensor] = None
def window_read_extent(forward_mode, spec_info, write_ids, batch_size: int) -> int:
"""How far past ``seq_lens`` a forward reads: the window it writes, when
its ``seq_lens`` do not count that window yet. A verify reads at most
``draft_token_num`` past them (a ragged verify writes fewer for some rows);
a draft decode reads back its own earlier steps; a draft extend's lengths
take its window only after its batch is built. Zero otherwise."""
if spec_info is None or write_ids is None or not batch_size:
return 0
if forward_mode.is_target_verify():
return spec_info.draft_token_num
if forward_mode.is_decode() or forward_mode.is_draft_extend_v2():
return write_ids.numel() // batch_size
return 0
class TokenSpan(msgspec.Struct, frozen=True):
"""A contiguous run ``[start, stop)`` of a write window's flat tokens (one
half of a two-batch-overlap split), which a reader takes as a view; a
``slice`` in `Cols` names columns of every row instead."""
start: int
stop: int
# Part of a `[batch, window]` write window: a slice of its columns, a
# contiguous run of its flat tokens, or a flat index into it (-1: the sink).
Cols = Union[slice, TokenSpan, torch.Tensor]
def pad_with_sink(ids: Optional[torch.Tensor], n: int) -> Optional[torch.Tensor]:
"""``ids`` padded to ``n`` entries with the sink (id 0 in every space), for
a batch whose write ids cover more lanes than its plan's (a padded batch,
a captured graph's buffer)."""
if ids is None or ids.shape[0] >= n:
return ids
padded = ids.new_zeros(n)
padded[: ids.shape[0]].copy_(ids)
return padded
class KVLocPlan:
"""The ids of one iteration, in every sub-pool, each computed once."""
def __init__(
self,
*,
source: KVIndexTranslator,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
):
self._source = source
self.req_pool_indices = req_pool_indices
# Committed lengths when the plan is built; every read this iteration
# stays within `seq_lens + read_extent` (the window it writes).
self.seq_lens = seq_lens
self.seq_lens_cpu = seq_lens_cpu
self.read_extent = read_extent
full = source.space(IdSpaceKind.FULL)
if write_slots is not None:
# A runner's own write buffer (graph capture, warmup): already in
# its pool's ids, naming the sink until a replay fills it, and kept
# by address by a captured graph.
assert write_virtual is None
self.write_virtual = None if full.write is not None else write_slots
self.write_physical = write_slots
else:
# Aliases the ScheduleBatch's tensor, which stays virtual for the
# radix tree, the accept path and lazy compaction's in-flight write
# set.
self.write_virtual = write_virtual
# FIXME: a speculative iteration builds its plan before its draft
# runs, so this launch sits on the host path ahead of the draft's
# graph, idle GPU time at low concurrency. Translate the window
# inside the iteration's first captured forward instead, into a
# capture-stable buffer the plan owns, which every later forward
# and direct writer reads -- once per sub-pool, the sliding-window
# ids (derived in `write_ids`) included.
self.write_physical = (
full.write(write_virtual)
if full.write is not None and write_virtual is not None
else write_virtual
)
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
full.key: self.write_physical
}
self._read_tables: Dict[Hashable, KVIndexTable] = {}
# -- writes ----------------------------------------------------------------
def bind(self, batch, reader: KVIndexTranslator, *, cols: Optional[Cols] = None):
"""Give ``batch`` (a ForwardBatch or a stand-in) this plan, the part of
the window it writes (``cols``), and its full-attention write ids in
`reader`'s pool. The one way a forward gets its write ids; another
sub-pool's come from `KVIndexTranslator.write_ids`."""
batch.kv_loc_plan = self
batch.kv_loc_cols = cols
batch.out_cache_loc = self.write_ids(reader, cols=cols)
batch.out_cache_loc_virtual = (
self.virtual_write_ids(cols=cols)
if self.is_translated_for(reader)
else None
)
batch.out_cache_loc_is_physical = batch.out_cache_loc is not None
def bind_replay(self, batch, reader: KVIndexTranslator, *, slots: torch.Tensor):
"""`bind`, for the batch a captured graph replays this iteration's
forward with. Its write ids are ``slots``, the runner's capture-stable
buffer that already holds this plan's write ids and names the sink past
them. The virtual mirror stays with the live batch."""
self.bind(batch, reader)
batch.out_cache_loc = slots
batch.out_cache_loc_virtual = None
def write_ids(
self,
reader: KVIndexTranslator,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
cols: Optional[Cols] = None,
) -> Optional[torch.Tensor]:
"""The write window as `reader`'s ``kind`` sub-pool indexes it, or None
when its pool has no such sub-pool. ``cols`` selects part of a
``[batch, window]`` window, for a forward that writes only part of it:
a slice of columns (a draft step ahead of the verify window it opens),
or a flat index into the window, -1 naming the sink (a ragged verify's
packed rows)."""
space = reader.space(kind)
if space is None:
return None
if space.key not in self._write_ids:
self._write_ids[space.key] = self._window_in(space)
return self._cols(self._write_ids[space.key], cols)
def _window_in(self, space: IdSpace) -> Optional[torch.Tensor]:
if self.write_virtual is None:
# No window this iteration, or a runner's own slots, which name
# the sink in every other sub-pool too.
if self.write_physical is None:
return None
return torch.zeros_like(self.write_physical)
if space.write is None:
return self.write_virtual
return space.write(self.write_virtual)
def virtual_write_ids(self, *, cols: Optional[Cols] = None):
"""The same columns in the virtual space: the ids the scheduler's
bookkeeping (lazy compaction's in-flight write set, the radix tree)
names."""
return self._cols(self.write_virtual, cols)
def _cols(self, ids: Optional[torch.Tensor], cols: Optional[Cols]):
if ids is None or cols is None:
return ids
if isinstance(cols, TokenSpan):
return ids[cols.start : cols.stop]
if isinstance(cols, torch.Tensor):
return torch.where(cols >= 0, ids[cols.clamp(min=0)], 0)
bs = int(self.req_pool_indices.numel())
if bs == 1:
return ids[cols]
return ids.view(bs, -1)[:, cols].reshape(-1)
def cols_slice(self, cols: Optional[Cols], tokens: slice) -> Cols:
"""The ``tokens`` of a forward that writes ``cols``: the columns of
one half of a batch split in two (two-batch overlap). A contiguous
half stays a `TokenSpan`; only columns spread over the rows become a
flat index into the window. Tokens past the window are left out; the
consumer pads them with the sink."""
if isinstance(cols, torch.Tensor):
return cols[tokens]
window = (
self.write_virtual
if self.write_virtual is not None
else self.write_physical
)
if cols is None or isinstance(cols, TokenSpan):
base = 0 if cols is None else cols.start
end = window.numel() if cols is None else cols.stop
start, stop, step = tokens.indices(end - base)
if step == 1:
return TokenSpan(base + start, base + max(start, stop))
index = torch.arange(window.numel(), device=window.device)
return self._cols(index, cols)[tokens]
def reads_from(
self,
source: KVIndexTranslator,
*,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
read_extent: int,
) -> KVLocPlan:
"""This plan's write ids, with reads planned over ``source``'s own
`req_to_token` rows and lengths -- a draft that reads a compact table
of its own while writing the iteration's window. The write ids are
shared, not translated again."""
plan = KVLocPlan(
source=source,
req_pool_indices=self.req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=None,
read_extent=read_extent,
)
plan.write_virtual = self.write_virtual
plan.write_physical = self.write_physical
plan._write_ids = self._write_ids
return plan
# -- reads -----------------------------------------------------------------
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
"""Whether a reader has had the ``kind`` table built yet."""
return self._source.space(kind).key in self._read_tables
def read_table(
self,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
rows: Optional[int] = None,
into: Optional[torch.Tensor] = None,
) -> KVIndexTable:
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
when a table reader first asks (in place in ``into``, that reader's
capture-stable table, when given) and shared by every later reader.
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
sink, appended by copying, never by translating again. Where reads stay
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
space = self._source.space(kind)
table = self._read_tables.get(space.key)
if table is None or (
table.is_translated and rows is not None and table.ids.shape[0] < rows
):
table = self._source.build_iteration_table(
self, space, rows=rows, previous=table, into=into
)
self._read_tables[space.key] = table
else:
assert into is None, "a destination goes to the table's first build"
if rows is not None and table.ids.shape[0] > rows:
# Built wider, for a captured graph's padded lanes: this reader's
# kernels size their batch by the table's rows.
table = msgspec.structs.replace(
table, ids=table.ids[:rows], row_ids=table.row_ids[:rows]
)
return table
def is_read_by(self, reader: KVIndexTranslator) -> bool:
"""Whether `reader` reads this plan's tables: it indexes the plan's
physical ids and gathers the same `req_to_token` rows. A reader with a
table of its own (a draft's compact `req_to_token`) plans its own
reads."""
return (
self.is_translated_for(reader)
and reader.req_to_token is self._source.req_to_token
)
def is_translated_for(self, reader: KVIndexTranslator) -> bool:
"""Whether `reader`'s pool indexes this plan's physical ids: the
target's, and a fused draft's, which lives in the target's full
sub-pool. A pass-through reader (a static pool, a private draft pool)
indexes the virtual ids."""
full = reader.space(IdSpaceKind.FULL)
return full.write is not None and full.key == self._full_key
@@ -5,45 +5,23 @@ The nesting, outermost first:
sub-pool a named pool inside the unified pool, with its own grow direction
and frontier. HOST is the role it plays when its entries also
carry draft bytes; only "full" hosts a draft here.
slot one token entry of that sub-pool. The draft's KV becomes extra
parts of every slot, after the host's own parts at
`draft_offset_in_entry()`, so ONE slot id per token covers target
and draft: one allocation, one free, one whole-page relocation
carry both.
slot one token entry of that sub-pool. The draft's KV is extra parts of
every slot, after the host's own parts, so ONE slot id per token
covers target and draft: one allocation, one free, one whole-page
relocation carry both.
region the draft geometry inside the host's entries. A region is not a
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
lane one position in the region's layer dimension -- one draft layer of
one runner, present in every slot.
Runners are the draft's execution copies, and the two head shapes place them
oppositely. A replicated head gives EVERY runner all the draft's layers, so
each needs its own lanes or the runners clobber each other's KV (3 runners x
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
either direction. Each runner owns a contiguous run of lanes, and the runs
tile the region in runner order.
A lane is not a layer. Runners are the draft's execution copies: a
replicated head gives EVERY runner all the draft's layers, on lanes of its own
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
while a per-depth head (one block per predicted position) gives each runner
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
contiguous run of lanes, and the runs tile the region in runner order.
Three phases, three moments in boot, and what carries each:
PLAN on the TARGET, before any memory exists
KVCacheConfigurator._fused_draft_decision()
draft_kv_profile(draft_model_config) what the checkpoint asks for
place_fused_draft(profile, runners) admit, or decline with a reason
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
CARVE as the byte buffer is cut
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
init_unified_*_pools(fused_draft=...)
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
UnifiedKVPool.build_dense_draft_views()
BIND on each DRAFT runner, built after the target
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
bind_fused_draft(...) -> UnifiedDraftKVPool
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
A draft whose layers need another sub-pool kind declines to a private pool.
A draft whose layers need another sub-pool kind declines fusion as a whole.
"""
from typing import List, Optional, Tuple
@@ -57,8 +35,8 @@ from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
sub-pool. The draft is a separate checkpoint, so its head geometry and
layer count differ from the host's; its K and V rows are two more parts of
the host entry, indexed by the host's physical token id."""
layer count can differ from the host's; its K and V rows are two more parts
of the host entry, indexed by the host's physical token id."""
lane_num: int
head_num: int
@@ -174,7 +152,6 @@ def draft_state_layer_num(draft_model_config) -> int:
mambaish = mambaish_config(draft_model_config)
if mambaish is None:
return 0
# Optional HF attribute: only conv-chain MTP configs declare it.
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
return 0
return len(mambaish.mamba2_cache_params.layers)
@@ -217,8 +194,8 @@ def draft_kv_profile(
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
private pool. Neither means fusion simply does not apply."""
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
fuse. Neither means fusion does not apply."""
placement: Optional[FusedDraftPlacement] = None
declined: Optional[str] = None
@@ -253,8 +230,8 @@ def place_fused_draft(
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host arm serves declines the whole draft to
its private pool."""
``"full"``. A layer kind no host sub-pool serves declines the whole
draft."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
+15 -20
View File
@@ -1871,26 +1871,21 @@ class KVWriteLoc:
All location info lives here (in the attention metadata), NOT in the pool:
- ``loc``: the generic per-token write location (``out_cache_loc``).
PHYSICAL on every pool: by allocation on non-unified pools,
rebound at ForwardBatch construction (``rebind_write_loc``) on
the unified pool.
PHYSICAL on every pool: by allocation on non-unified pools, the
iteration's plan's write ids on the unified pool (``KVLocPlan.bind``).
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
otherwise); under the unified pool the translator derives it from the
same rebound loc (``sliding_window_write_loc_for``).
- ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the
construction-time rebind it is the SAME id space as ``loc``, so pools
fall back to ``loc`` when it is ``None`` -- only triton's captured path
still passes its capture-stable
``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a
same-space alias slated for collapse).
otherwise): the same plan's write ids in the sliding-window sub-pool
(``KVIndexTranslator.write_ids``).
- ``full_loc``: OPTIONAL full-attention-sub-pool location, in the same id
space as ``loc``; pools fall back to ``loc`` when it is ``None``.
``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved
loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``);
``loc`` is the generic fallback. Bundling them lets a backend issue one
``set_kv_buffer`` call regardless of pool type.
``physical`` marks the locs as physical token ids: the batch's write loc
after ``rebind_write_loc``, or ids a backend translated itself. A unified
``physical`` marks the locs as physical token ids: the write ids a plan
bound to the batch, or ids produced separately and marked so. A unified
pool's write door refuses a loc not marked physical; other pools do not
check.
"""
@@ -1910,9 +1905,9 @@ class KVWriteLoc:
) -> KVWriteLoc:
"""The batch's ``out_cache_loc`` as a write loc, carrying the batch's
physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under
the same mark, so it must be derived from that rebound loc (as
``sliding_window_write_loc_for`` does); a loc produced separately
states its own mark with ``KVWriteLoc(loc, physical=...)``."""
the same mark, so it must come from the same plan (as
``KVIndexTranslator.write_ids`` does); a loc produced separately states
its own mark with ``KVWriteLoc(loc, physical=...)``."""
return cls(
forward_batch.out_cache_loc,
swa_loc,
@@ -2095,9 +2090,9 @@ class KVCache(abc.ABC):
if self.requires_physical_write_loc and not write_loc_is_physical(loc_info):
raise ValueError(
f"{where}: write loc is not marked physical. Hand the pool "
"KVWriteLoc.for_batch(forward_batch) after "
"KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, "
"physical=True) for ids translated separately."
"KVWriteLoc.for_batch(forward_batch) for a batch bound to a "
"plan (KVLocPlan.bind), or KVWriteLoc(loc, physical=True) for "
"ids translated separately."
)
@abc.abstractmethod
@@ -4734,7 +4729,7 @@ class MLATokenToKVPool(KVCache):
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
# False: this pool takes a WIDENED loc. The unified pool resolves it in
# `KVIndexTranslator.rebind_write_loc` and flips this.
# the plan's write translation (`translate_write_loc`) and flips this.
write_loc_is_dcp_resolved = False
@property
@@ -15,15 +15,11 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
"""Dense draft KV over the draft parts of one host sub-pool's entries.
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
``layer_lanes`` maps each of this runner's layer ids to its region lane.
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
produced by the allocator's translate (the id-space choke point binds it,
see KVIndexTranslator); the pool exposes `host_allocator` for that
binding. Relocation needs no method here: compaction moves whole page
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
raises so a stray per-slot move fails loudly instead of corrupting the
fused layout.
every slot of the host sub-pool; ``layer_lanes`` maps each of this
runner's layer ids to its region lane. Locs are the target's PHYSICAL
token ids, translated through `host_allocator`. Compaction moves whole
page envelopes on the HOST pool, draft bytes included, so `move_kv_cache`
raises rather than let a per-slot move corrupt the fused layout.
"""
requires_physical_write_loc = True
@@ -149,8 +149,7 @@ class MHASubPoolSpec(SubPoolSpec):
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
The slot stride stays the one entry, so host and draft views are indexed
by the same physical token id; only their offsets differ. `draft_region
is None` keeps the layout byte-identical to the unfused one.
by the same physical token id; only their offsets differ.
"""
head_num: int
@@ -930,8 +929,9 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
# Lifetime owned by UnifiedKVPool; do not delete the views.
pass
# `rebind_write_loc` already collapsed the widened id and sent the rows this
# rank does not own to the padding sink.
# The plan's write translation (`translate_write_loc`) already collapsed
# the widened id and sent the rows this rank does not own to the padding
# sink.
write_loc_is_dcp_resolved = True
def get_kv_size_bytes(self):
@@ -814,7 +814,6 @@ class CPUGraphRunner:
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -832,6 +831,7 @@ class CPUGraphRunner:
num_token_non_padded=self.num_token_non_padded,
global_forward_mode=self.capture_forward_mode,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
# Wrap all forward calls with capture_with_skip_cross_attention so that
# mllama (and any other encoder-decoder model) sees the correct compile-
# time constant for skip_cross_attention during tracing.
@@ -83,6 +83,7 @@ class ForwardBatchDeepSeekMHAMixin:
):
self.prefix_chunk_kv_indices = []
req_to_token = get_req_to_token_pool().req_to_token
translator = get_attn_backend().kv_index_translator
for idx in range(self.num_prefix_chunks):
chunk_starts = self.prefix_chunk_starts[idx]
chunk_seq_lens = self.prefix_chunk_seq_lens[idx]
@@ -93,6 +94,18 @@ class ForwardBatchDeepSeekMHAMixin:
num_chunk_tokens, dtype=torch.int32, device=device
)
if translator.reads_are_translated:
# The plan's table already holds the chunk's physical ids.
translator.pack_read_stream(
self.kv_loc_plan,
req_pool_indices=self.req_pool_indices,
seq_lens=chunk_seq_lens,
indptr=chunk_cu_seq_lens,
out=chunk_kv_indices,
kv_start_idx=chunk_starts,
)
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
continue
create_chunked_prefix_cache_kv_indices[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
@@ -107,7 +120,6 @@ class ForwardBatchDeepSeekMHAMixin:
self.prefix_chunk_starts_cpu[idx],
self.prefix_chunk_seq_lens_cpu[idx],
)
translator = get_attn_backend().kv_index_translator
chunk_kv_indices = translator.translate_dcp_read_ids(chunk_kv_indices)
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
@@ -266,19 +278,28 @@ class ForwardBatchDeepSeekMHAMixin:
device=self.req_pool_indices.device,
)
kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0)
req_to_token = get_req_to_token_pool().req_to_token
create_flashinfer_kv_indices_triton[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
self.seq_lens,
kv_indptr,
None,
kv_indices,
req_to_token.shape[1],
)
# None on a backend that never bound a translator.
# None on a backend that never bound a translator, which reads virtual
# ids; otherwise one gather from the iteration's plan (DCP takes its
# own branch upstream).
src = get_attn_backend().kv_index_translator
if src is not None:
kv_indices = src.translate_full_attn_ids(kv_indices)
if src is None:
req_to_token = get_req_to_token_pool().req_to_token
create_flashinfer_kv_indices_triton[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
self.seq_lens,
kv_indptr,
None,
kv_indices,
req_to_token.shape[1],
)
else:
src.pack_read_stream(
self.kv_loc_plan,
req_pool_indices=self.req_pool_indices,
seq_lens=self.seq_lens,
indptr=kv_indptr,
out=kv_indices,
)
self.mha_one_shot_kv_indices = kv_indices
return kv_indices
@@ -43,7 +43,6 @@ from sglang.srt.environ import envs
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
compute_req_all_ids_info,
)
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_slot_in,
@@ -77,6 +76,7 @@ if TYPE_CHECKING:
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
@@ -192,21 +192,6 @@ def _elastic_should_preserve_local_token_counts(
return uneven_token_count
def _localize_npu_dcp_out_cache_loc(
out_cache_loc: torch.Tensor,
*,
interleave_size: int,
) -> torch.Tensor:
"""Map allocator-global NPU DCP slots to this target rank."""
parallel = get_parallel()
return localize_dcp_indices(
out_cache_loc,
parallel.dcp_size,
parallel.dcp_rank,
interleave_size,
)
class ForwardMode(IntEnum):
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
# It is also called "prefill" in common terminology.
@@ -532,14 +517,23 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# The original sequence length without being chunked. Qwen-1M related.
orig_seq_lens: Optional[torch.Tensor] = None
# The write loc before `rebind_write_loc` replaced it with physical
# ids; a backend re-derives from it into its capture-stable buffer.
# This iteration's KV ids, translated once per sub-pool (`KVLocPlan`).
# Every forward of the iteration holds the same plan; `out_cache_loc` is
# its write window in the full-attention ids this runner's pool indexes,
# and a backend that writes another sub-pool asks the plan for its ids
# (`KVIndexTranslator.write_ids`).
kv_loc_plan: Optional[KVLocPlan] = None
# The part of the plan's window this forward writes (None: all of it).
kv_loc_cols: Optional[Cols] = None
# The same write ids in the virtual space when `out_cache_loc` holds
# translated ones (None on a pool that indexes virtual ids), for the
# scheduler's bookkeeping: lazy compaction's in-flight write set, TBO's
# split, state capture.
out_cache_loc_virtual: Optional[torch.Tensor] = None
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
# consumed by the Ascend backend for PA_ND block tables. None elsewhere.
out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None
# Whether `out_cache_loc` holds physical ids: set by
# KVIndexTranslator.rebind_write_loc; capture-time batches declare it.
# Whether `out_cache_loc` holds physical ids: set by `KVLocPlan.bind`.
out_cache_loc_is_physical: bool = False
# The indices to track mamba state with
mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64
@@ -937,9 +931,16 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
return_hidden_states_before_norm: bool,
extend_position_info=None,
spec_mrope_positions: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
write_cols: Optional[Cols] = None,
):
# init_new must not mutate the input ScheduleBatch; per-forward
# overrides go through explicit keyword arguments.
#
# `kv_loc_plan`: the iteration's plan when this forward is one of
# several over the same slots (a speculative iteration), `write_cols`
# the columns of its window this forward writes. Without one, this
# forward is its own iteration and builds the plan itself.
# capture_hidden_mode=None means no override: capture the server's
# configured maximum so lower-mode requests can share one graph.
@@ -1051,15 +1052,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
spec_info=batch.spec_info,
)
# ScheduleBatch and req_to_token keep allocator-global slot identities.
# Preserve that view before exposing rank-local NPU DCP write slots.
# ScheduleBatch and req_to_token keep allocator-global slot identities,
# and so does the plan's window. An NPU DCP target writes rank-local
# slots, which its translator maps the window to at `bind` below; keep
# the global view for the consumers that read it.
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
ret.origin_out_cache_loc = ret.out_cache_loc
if ret.out_cache_loc is not None:
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
ret.out_cache_loc,
interleave_size=model_runner.page_size,
)
ret._maybe_init_non_generation_fields(batch)
device = model_runner.device
@@ -1073,7 +1071,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
device,
)
model_runner.kv_index_translator.rebind_write_loc(ret)
translator = model_runner.kv_index_translator
if kv_loc_plan is None:
kv_loc_plan = translator.own_plan(ret)
kv_loc_plan.bind(ret, translator, cols=write_cols)
if envs.SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.get():
hashed = _hash_rids_to_tensor(
@@ -2143,6 +2144,8 @@ def build_inner_fb_view(
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
# A caller may hand in another view that does not carry this field.
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
kv_loc_plan=getattr(forward_batch, "kv_loc_plan", None),
kv_loc_cols=getattr(forward_batch, "kv_loc_cols", None),
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
spec_info=forward_batch.spec_info,
@@ -898,6 +898,7 @@ class ModelRunner:
token_to_kv_pool=self.token_to_kv_pool,
page_size=self.page_size or 1,
device=self.device,
is_draft_worker=self.is_draft_worker,
)
def max_shared_logits_buffer_rows(self) -> int:
@@ -662,7 +662,6 @@ class BaseRunner(ABC):
forward_batch = ForwardBatch(
forward_mode=capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=batch_size,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -698,6 +697,7 @@ class BaseRunner(ABC):
global_forward_mode=capture_forward_mode,
lora_ids=lora_ids,
)
mr.kv_index_translator.bind_runner_slots(forward_batch)
if buffers.ngram_embedding_info is not None:
forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice(
@@ -189,6 +189,8 @@ def build_replay_fb_view(
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
max_seq_len_override=forward_batch.max_seq_len_override,
@@ -1005,7 +1007,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -1043,6 +1044,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
rids_int=rids_int,
bootstrap_room_ids_int=bootstrap_room_ids_int,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
# Trip the coordinator so the hisparse code path is captured into the
# graph; backends read it from self.model_runner.hisparse_coordinator.
@@ -1118,9 +1120,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self._post_process_after_profile(prof)
self._profiler = None
# No pool-side pin to clear: the captured full-physical write loc rides the
# backend's `ForwardMetadata.out_cache_loc_full_physical` (-> KVWriteLoc.full_loc).
def _capture_one_stream(self, stream_idx: Optional[int] = None) -> None:
avail_mem = get_available_gpu_memory(
self.model_runner.device,
@@ -1271,9 +1270,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
if (c := self.model_runner.canary_manager) is not None
else contextlib.nullcontext()
)
# Full-physical write loc lives in the attention metadata (the backend's
# `out_cache_loc_full_physical` -> KVWriteLoc.full_loc), so the runner
# wires no buffer here. (SWA write loc rides the `swa_out_cache_loc` rail.)
with canary_ctx:
shape_key = self._make_graph_key(
@@ -1496,7 +1496,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
with torch.device(self.device):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.EXTEND,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=_slot("input_ids"),
input_embeds=(
@@ -1562,6 +1561,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
max_seq_len_override=self.max_context_size,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
ngram_manager = self.model_runner.ngram_embedding_manager
if ngram_manager.enabled:
forward_batch.ngram_embedding_info = NgramEmbeddingInfo.create(
@@ -1861,7 +1861,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
static_forward_batch = ForwardBatch(
forward_mode=pcg_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
input_embeds=input_embeds,
@@ -1926,6 +1925,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
),
max_seq_len_override=self.max_context_size,
)
forward_batch.kv_loc_plan.bind_replay(
static_forward_batch,
self.model_runner.kv_index_translator,
slots=out_cache_loc,
)
# The n-gram hasher runs outside the graph and reads this at replay.
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
static_forward_batch.engram_history = forward_batch.engram_history
+2 -2
View File
@@ -803,8 +803,8 @@ class DSparkDraftMixin:
attn.attn.v_scale,
)
else:
# Translated by the injector (physical by allocation on a plain
# pool).
# `cache_loc` comes from `TargetHiddenKvInjector.ids_for`,
# already in the ids the draft pool indexes.
pool.set_kv_buffer(
attn.attn,
KVWriteLoc(cache_loc, physical=True),
@@ -481,12 +481,8 @@ class InklingAttention(nn.Module):
do_store = do_bf16_store or do_mxfp8_store
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
# SWA sub-pool + the backend's full->SWA translated write location.
# SWA sub-pool + the plan's sliding-window write ids (backend buffer).
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
es = q.element_size()
@@ -584,10 +580,6 @@ class InklingAttention(nn.Module):
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
is_v2 = forward_batch.forward_mode.is_draft_extend_v2()
@@ -691,10 +683,6 @@ class InklingAttention(nn.Module):
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
es = q.element_size()
+15 -7
View File
@@ -7,6 +7,7 @@ import torch
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import Cols, IdSpaceKind, KVLocPlan
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
@@ -51,6 +52,10 @@ class DFlashVerifyInput(SpecInput):
live_seq_lens_cpu: Optional[torch.Tensor] = None
# Conservative request-lifetime bound for candidate graph dispatch.
candidate_max_seq_len_upper_bound: Optional[int] = None
# The iteration's plan when the caller planned the verify window, and the
# part of it this verify writes (`KVLocPlan.write_ids`).
kv_loc_plan: Optional[KVLocPlan] = None
kv_loc_cols: Optional[Cols] = None
def __post_init__(self):
super().__init__(spec_input_type=SpecInputType.DFLASH_VERIFY)
@@ -101,6 +106,8 @@ class DFlashVerifyInput(SpecInput):
target_worker.model_runner,
capture_hidden_mode=self.capture_hidden_mode,
return_hidden_states_before_norm=False,
kv_loc_plan=self.kv_loc_plan,
write_cols=self.kv_loc_cols,
)
can_run_cuda_graph = bool(
@@ -133,13 +140,14 @@ class DFlashVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
plan: KVLocPlan,
kv_start_idx: Optional[torch.Tensor] = None,
kv_indices_buf: Optional[torch.Tensor] = None,
sliding_window: bool = False,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
device = req_pool_indices.device
bs = req_pool_indices.numel()
@@ -179,14 +187,14 @@ class DFlashVerifyInput(SpecInput):
dtype=torch.int32,
device=device,
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=sliding_window,
kind=kind,
)
mask = self.custom_mask
if mask is not None:
@@ -26,6 +26,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
@@ -1744,6 +1745,11 @@ class DFlashWorkerV2(BaseSpecWorker):
return out_tokens
def _draft_write_ids(self, plan: KVLocPlan) -> torch.Tensor:
"""The plan's write ids as the draft pool indexes them: the target's
physical ids for a fused draft, the virtual ids for a private one."""
return plan.write_ids(self.draft_model_runner.kv_index_translator)
def _append_target_hidden_to_draft_kv_by_loc(
self,
*,
@@ -1845,17 +1851,9 @@ class DFlashWorkerV2(BaseSpecWorker):
if commit_lens.dtype != torch.int32:
commit_lens = commit_lens.to(torch.int32)
# On a translating pool the translate writes a new tensor, so the
# callers' locs stay virtual: the post-verify 2-D buffer is re-read as
# virtual ids by the compact req_to_token rebuild. On a plain pool it
# returns `cache_loc` itself.
translator = self.draft_model_runner.kv_index_translator
cache_loc = translator.translate_full_attn_ids(cache_loc)
if cache_loc_2d is not None:
# Same ids, so reshape the translated copy rather than translate
# them a second time.
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
# `cache_loc` arrives in the draft pool's id space (`_draft_write_ids`);
# the callers' window buffers stay virtual for the compact
# req_to_token rebuild.
with (
torch.inference_mode(),
draft_tp_context(self.draft_owns_attention),
@@ -2265,7 +2263,9 @@ class DFlashWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=CaptureHiddenMode.FULL,
return_kv_loc_plan=True,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
logits_output, next_token_ids = (
batch_output.logits_output,
@@ -2316,9 +2316,10 @@ class DFlashWorkerV2(BaseSpecWorker):
ctx_lens,
int(sum(batch.extend_lens)),
)
# The draft KV goes to the slots the target prefill just wrote.
self._append_target_hidden_to_draft_kv_by_loc(
target_hidden=logits_output.hidden_states,
cache_loc=batch.out_cache_loc,
cache_loc=self._draft_write_ids(kv_loc_plan),
positions=positions,
extend_lens=ctx_lens,
)
@@ -2494,6 +2495,19 @@ class DFlashWorkerV2(BaseSpecWorker):
positions = positions_2d.reshape(-1)
verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1)
# The iteration's window -- the draft block, the verify and the
# target-hidden KV writes all write it -- planned once.
kv_loc_plan = self.target_worker.model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=(
batch.seq_lens_cpu
if batch.seq_lens_cpu is not None
else draft_input.nxt_kv_lens_cpu
),
write_virtual=verify_out_cache_loc,
read_extent=block_size,
)
seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs]
if self.use_compact_draft_cache:
@@ -2555,10 +2569,20 @@ class DFlashWorkerV2(BaseSpecWorker):
),
global_num_token_non_padded_cpu=bs * block_size,
)
# Hand-built draft batch bypasses ForwardBatch.init_new: under the
# unified pool the write loc must be rebound to the draft's
# kernel-facing ids here (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(forward_batch)
# The draft block writes the whole window. A compact draft reads its
# own `req_to_token` rows over its own lengths.
draft_translator = self.draft_model_runner.kv_index_translator
draft_plan = (
kv_loc_plan.reads_from(
draft_translator,
seq_lens=draft_seq_lens,
seq_lens_cpu=seq_lens_cpu,
read_extent=block_size,
)
if self.use_compact_draft_cache
else kv_loc_plan
)
draft_plan.bind(forward_batch, draft_translator)
if self.selector is not None or self.lilicorr is not None:
self._selector_sample = None
@@ -2713,6 +2737,7 @@ class DFlashWorkerV2(BaseSpecWorker):
draft_token_num=int(self.block_size),
custom_mask=custom_mask,
capture_hidden_mode=CaptureHiddenMode.FULL,
kv_loc_plan=kv_loc_plan,
)
batch.out_cache_loc = verify_out_cache_loc
@@ -2877,10 +2902,11 @@ class DFlashWorkerV2(BaseSpecWorker):
# Consume in this step: every decode graph size shares one aux output,
# which the next target forward overwrites (resolve_aux_hidden_states_width).
cache_loc = self._draft_write_ids(kv_loc_plan)
self._append_target_hidden_to_draft_kv_by_loc(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
cache_loc=verify_out_cache_loc,
cache_loc_2d=verify_out_cache_loc_2d,
cache_loc=cache_loc,
cache_loc_2d=cache_loc.view(verify_out_cache_loc_2d.shape),
positions=positions,
commit_lens=commit_lens,
)
@@ -360,7 +360,7 @@ class DraftBlockProposer:
)
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
# loc to the draft's kernel-facing ids (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(idle_batch)
self.draft_model_runner.kv_index_translator.bind_own_plan(idle_batch)
self._fill_dp_moe_sync_metadata(idle_batch, batch)
with torch.inference_mode():
self.draft_model_runner.forward(idle_batch)
@@ -435,10 +435,10 @@ class DraftBlockProposer:
),
global_num_token_non_padded_cpu=draft_num_tokens,
)
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
# loc to the draft's kernel-facing ids (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(
draft_forward_batch
verify_window.kv_loc_plan.bind(
draft_forward_batch,
self.draft_model_runner.kv_index_translator,
cols=slice(0, query_token_num),
)
self._fill_dp_moe_sync_metadata(draft_forward_batch, batch)
graph_runner = self.draft_model_runner.decode_cuda_graph_runner
@@ -5,12 +5,12 @@ import torch
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
)
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
BuildCommitInjectLayout,
build_unified_commit_inject_layout,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
@@ -32,6 +32,11 @@ class TargetHiddenKvInjector:
self.verify_num_draft_tokens = verify_num_draft_tokens
self._block_pos_offsets = block_pos_offsets
def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor:
"""The plan's write ids as the draft pool indexes them: the target's
physical ids for a fused draft, the virtual ids for a private one."""
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
def inject_target_hidden(
self,
*,
@@ -70,16 +75,7 @@ class TargetHiddenKvInjector:
device=device, dtype=torch.int64, non_blocking=True
)
# On a translating pool the translate writes a new tensor, so the
# callers' locs stay virtual; on a plain pool it returns `cache_loc`
# itself.
translator = self.draft_model_runner.kv_index_translator
cache_loc = translator.translate_full_attn_ids(cache_loc)
if cache_loc_2d is not None:
# Callers pass `cache_loc` viewed [bs, verify_len]: reshape the
# translated copy rather than translate the same ids twice.
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
# `cache_loc` arrives in the draft pool's id space (`ids_for`).
pool = self.draft_model_runner.token_to_kv_pool
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
if target_hidden_is_projected:
@@ -214,6 +210,7 @@ class TargetHiddenKvInjector:
hidden_strided: torch.Tensor,
commit_lens: torch.Tensor,
bs: int,
kv_loc_plan: KVLocPlan,
) -> None:
stride = self.verify_num_draft_tokens
prefix_lens = batch.seq_lens
@@ -252,15 +249,7 @@ class TargetHiddenKvInjector:
return
positions_2d = prefix_lens.unsqueeze(1) + self._block_pos_offsets
verify_cache_loc = assign_extend_cache_locs_func(
req_pool_indices=batch.req_pool_indices,
req_to_token=self.model_runner.req_to_token_pool.req_to_token,
start_offset=prefix_lens,
end_offset=prefix_lens + stride,
batch_size=bs,
draft_token_num=stride,
device=self.device,
)
verify_cache_loc = self.ids_for(kv_loc_plan)
verify_cache_loc_2d = verify_cache_loc.view(bs, stride)
self.inject_target_hidden(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
@@ -20,6 +20,7 @@ from sglang.srt.managers.overlap_utils import (
ResolvedConfidence,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments
@@ -65,6 +66,9 @@ class VerifyWindow(msgspec.Struct, frozen=True):
positions_2d: torch.Tensor
verify_cache_loc: torch.Tensor
verify_cache_loc_2d: torch.Tensor
# The iteration's plan of `verify_cache_loc`: the draft block, the verify
# and the target-hidden KV writes all take their ids from it.
kv_loc_plan: KVLocPlan
class DSparkVerifyPlanner:
@@ -827,7 +831,10 @@ def alloc_verify_window(
verify_num_draft_tokens: int,
block_pos_offsets: torch.Tensor,
model_runner,
seq_lens_cpu: Optional[torch.Tensor],
) -> VerifyWindow:
"""``seq_lens_cpu``: a host bound on ``batch.seq_lens``, which sizes the
plan's read table."""
prefix_lens = batch.seq_lens
verify_w = verify_num_draft_tokens
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
@@ -845,6 +852,13 @@ def alloc_verify_window(
positions_2d=positions_2d,
verify_cache_loc=verify_cache_loc,
verify_cache_loc_2d=verify_cache_loc_2d,
kv_loc_plan=model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=verify_cache_loc,
read_extent=verify_w,
),
)
@@ -312,6 +312,7 @@ class TargetVerifyExecutor:
custom_mask=None,
capture_hidden_mode=CaptureHiddenMode.FULL,
live_seq_lens_cpu=batch.seq_lens_cpu,
kv_loc_plan=verify_window.kv_loc_plan,
)
batch.out_cache_loc = verify_cache_loc
seq_lens_cpu_backup = batch.seq_lens_cpu
@@ -388,6 +389,7 @@ class TargetVerifyExecutor:
hidden_strided=hidden_strided,
commit_lens=commit_lens,
bs=bs,
kv_loc_plan=verify_window.kv_loc_plan,
)
return
hidden = logits_output.hidden_states
@@ -404,10 +406,11 @@ class TargetVerifyExecutor:
state_slot = (
batch.req_pool_indices[:bs].view(-1, 1).expand(bs, vlen).reshape(-1)
)
cache_loc = self.kv_injector.ids_for(verify_window.kv_loc_plan)
self.kv_injector.inject_target_hidden(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
cache_loc=verify_window.verify_cache_loc,
cache_loc_2d=verify_window.verify_cache_loc_2d,
cache_loc=cache_loc,
cache_loc_2d=cache_loc.view(verify_window.verify_cache_loc_2d.shape),
positions=verify_window.positions_2d.reshape(-1),
commit_lens=commit_lens,
state_slot=state_slot,
@@ -420,6 +423,7 @@ class TargetVerifyExecutor:
layout: RaggedVerifyLayout,
ragged_window: RaggedVerifyWindow,
sampling_info,
kv_loc_plan,
) -> TargetVerifyResult:
verify_input = DFlashVerifyInput(
draft_token=ragged_window.verify_ids,
@@ -429,6 +433,9 @@ class TargetVerifyExecutor:
capture_hidden_mode=CaptureHiddenMode.FULL,
ragged_verify_layout=layout,
live_seq_lens_cpu=batch.seq_lens_cpu,
# The packed rows are a selection of the planned verify window.
kv_loc_plan=kv_loc_plan,
kv_loc_cols=ragged_window.window_index,
)
batch.out_cache_loc = ragged_window.verify_cache_loc
seq_lens_cpu_backup = batch.seq_lens_cpu
@@ -461,6 +468,7 @@ class TargetVerifyExecutor:
bs: int,
device: str,
sampling_info,
verify_window: VerifyWindow,
inject_gate: bool = False,
) -> tuple[TargetVerifyResult, torch.Tensor]:
ragged_window = BuildRaggedVerifyWindow.execute(
@@ -480,6 +488,7 @@ class TargetVerifyExecutor:
layout=layout,
ragged_window=ragged_window,
sampling_info=sampling_info,
kv_loc_plan=verify_window.kv_loc_plan,
)
logits_output = target_verify.logits_output
@@ -642,6 +642,7 @@ class DSparkWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=CaptureHiddenMode.FULL,
return_kv_loc_plan=True,
)
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
target_hidden_is_projected = (
@@ -705,7 +706,9 @@ class DSparkWorkerV2(BaseSpecWorker):
repeats,
output_size=num_tokens,
)
cache_loc = batch.out_cache_loc
# The draft KV goes to the slots the target prefill just wrote.
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
batch_output.kv_loc_plan = None
token_indices = logits_output.hidden_states_token_indices
if token_indices is not None:
cache_loc = cache_loc[token_indices]
@@ -842,6 +845,11 @@ class DSparkWorkerV2(BaseSpecWorker):
verify_num_draft_tokens=self.verify_num_draft_tokens,
block_pos_offsets=self._block_pos_offsets,
model_runner=self.model_runner,
seq_lens_cpu=(
batch.seq_lens_cpu
if batch.seq_lens_cpu is not None
else draft_input.nxt_kv_lens_cpu
),
)
sampling_info = batch.sampling_info
@@ -936,6 +944,7 @@ class DSparkWorkerV2(BaseSpecWorker):
bs=bs,
device=device,
sampling_info=sampling_info,
verify_window=verify_window,
inject_gate=fold_eligible,
)
else:
@@ -459,7 +459,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=num_seqs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -485,6 +484,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL
),
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
def run_once():
self.draft_attn_backend.init_forward_metadata_in_graph(forward_batch)
@@ -424,7 +424,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -448,6 +447,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
if self.buffers.dsa_seed_topk_capture is not None:
spec_info.dsa_seed_topk_capture = self.buffers.dsa_seed_topk_capture[
@@ -649,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
out_cache_loc=buffers.out_cache_loc[:num_tokens],
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
+19 -13
View File
@@ -5,6 +5,7 @@ from typing import Callable, List, Optional
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -34,6 +35,9 @@ class EagleVerifyInput(SpecInput):
draft_probs: torch.Tensor = None
prepared_out_cache_loc: Optional[torch.Tensor] = None
prepared_mrope_positions: Optional[torch.Tensor] = None
# The iteration's plan when the draft planned the verify window
# (`prepared_out_cache_loc`); verify and draft extend take it.
kv_loc_plan: Optional[KVLocPlan] = None
# Shape info for padding
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
@@ -90,11 +94,12 @@ class EagleVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
device = req_pool_indices.device
batch_size = req_pool_indices.numel()
qo_indptr = torch.arange(
@@ -113,13 +118,13 @@ class EagleVerifyInput(SpecInput):
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
mask_numel = (
paged_kernel_lens_sum * self.draft_token_num
@@ -402,10 +407,11 @@ class EagleDraftExtendInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: Optional[int],
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""Draft-extend CSR args. The lens already include the window, so
unlike verify nothing is widened here."""
"""Draft-extend CSR args. ``paged_kernel_lens`` already includes the
draft-extend window."""
device = req_pool_indices.device
bs = self.num_correct_drafts.numel()
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
@@ -426,12 +432,12 @@ class EagleDraftExtendInput(SpecInput):
paged_kernel_lens_sum, dtype=torch.int32, device=device
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
return kv_indices, cum_kv_seq_len, qo_indptr, None
@@ -607,6 +607,7 @@ def eagle_prepare_for_verify(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=verify_input.prepared_mrope_positions,
kv_loc_plan=verify_input.kv_loc_plan,
)
# Run attention backend plan and cuda graph preparation
@@ -55,6 +55,8 @@ if _is_cpu:
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
@@ -121,7 +123,10 @@ def prepare_for_draft_extend(
return_hidden_states_before_norm: bool,
widened_out_cache_loc: Optional[torch.Tensor] = None,
widened_positions: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""``kv_loc_plan``: the iteration's plan, whose window the draft extend
writes again (the verify window); a widened window plans its own."""
bs = len(batch.seq_lens)
# Optional window widening (num_front_tokens=0 -> off): prepend that many
# rows below the boundary. Locs/positions arrive precomputed; token/hidden
@@ -204,6 +209,7 @@ def prepare_for_draft_extend(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=return_hidden_states_before_norm,
extend_position_info=extend_position_info,
kv_loc_plan=None if widen else kv_loc_plan,
)
# Forward sees post-write length (draft extend writes num_draft_tokens
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
@@ -243,18 +249,35 @@ def prepare_for_draft(
draft_model_runner: ModelRunner,
topk: int,
num_steps: int,
):
*,
target_translator: Optional[KVIndexTranslator] = None,
num_draft_tokens: Optional[int] = None,
) -> tuple[ForwardBatch, bool, Optional[KVLocPlan]]:
"""The draft-decode batch. With ``target_translator``, a chain draft plans
the iteration's whole write window (the ``num_draft_tokens`` slots past
``seq_lens`` that verify and draft extend also write) and writes its first
``num_steps`` columns; that plan is returned, else None."""
prepared_positions = None
prepared_mrope_positions = None
kv_loc_plan = None
if not batch.forward_mode.is_idle():
bs = len(batch.seq_lens)
# Assign cache locations (draft-write targets).
page_size = batch.token_to_kv_pool_allocator.page_size
plans_window = (
target_translator is not None
and topk == 1
and num_draft_tokens is not None
and num_draft_tokens > num_steps
and not _is_cpu
)
# A chain draft's slots lead the iteration's window: gather the whole
# window in the same launch.
window_cols = num_draft_tokens if plans_window else num_steps
if page_size == 1 or topk == 1:
batch.out_cache_loc = torch.empty(
(bs * topk * num_steps,),
(bs * topk * window_cols,),
dtype=torch.int64,
device=batch.device,
)
@@ -285,13 +308,21 @@ def prepare_for_draft(
batch.out_cache_loc,
req_to_token_pool.req_to_token.shape[1],
topk,
num_steps,
window_cols,
positions=prepared_positions,
mrope=prepared_mrope_positions,
BS=bs,
WRITE_POSITIONS=prepared_positions is not None,
WRITE_MROPE=prepared_mrope_positions is not None,
)
if plans_window:
kv_loc_plan = target_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=batch.seq_lens,
seq_lens_cpu=batch.seq_lens_cpu,
write_virtual=batch.out_cache_loc,
read_extent=num_draft_tokens,
)
else:
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
# Reduce out_cache_loc from the page-aligned tree region down to the
@@ -352,11 +383,13 @@ def prepare_for_draft(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=prepared_mrope_positions,
kv_loc_plan=kv_loc_plan,
write_cols=None if kv_loc_plan is None else slice(0, num_steps),
)
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
forward_batch
)
return forward_batch, can_run_decode_cuda_graph
return forward_batch, can_run_decode_cuda_graph, kv_loc_plan
def build_eagle_verify_input(
@@ -41,6 +41,7 @@ from sglang.srt.layers.moe.utils import (
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
@@ -606,7 +607,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def draft(self, batch: ScheduleBatch, *, with_topology: bool = False):
draft_input: EagleDraftInput = batch.spec_info
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
draft_input,
self.req_to_token_pool,
batch,
@@ -614,6 +615,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
self.topk,
self.speculative_num_steps,
target_translator=self.target_worker.model_runner.kv_index_translator,
num_draft_tokens=self.speculative_num_draft_tokens,
)
if (
can_run_decode_cuda_graph
@@ -673,6 +676,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
tree_mask_mode=self.tree_mask_mode,
device=self.device,
)
if kv_loc_plan is not None:
# Verify and draft extend write the window the draft planned.
verify_input.kv_loc_plan = kv_loc_plan
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
if with_topology:
# PP+spec relays the tree so every stage rebuilds the same verify
# input; the mask build needs the topology this one was built from.
@@ -900,6 +907,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
target_hidden_states: torch.Tensor,
next_token_ids: torch.Tensor,
mm_input_embeds: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""
Run draft model extend to correctly fill the KV cache.
@@ -908,6 +916,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
batch: The batch to run.
target_hidden_states: Hidden states from the target model forward
next_token_ids: Next token ids generated from the target forward.
kv_loc_plan: The target prefill's plan; the draft writes the same
slots.
"""
# Construct input_ids
if not batch.forward_mode.is_idle():
@@ -959,6 +969,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=False,
kv_loc_plan=kv_loc_plan,
)
forward_batch.return_logprob = False
if mm_input_embeds is not None:
@@ -1035,7 +1046,11 @@ class EagleDraftWorker(EagleDraftWorkerBase):
return buf[:num_tokens]
def _draft_extend_for_decode(
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
self,
batch: ScheduleBatch,
batch_result: GenerationBatchResult,
*,
kv_loc_plan: Optional[KVLocPlan] = None,
):
# Cast to int64 before entering plan stream to avoid cross-stream
# synchronization issues with .to() inside the plan stream context.
@@ -1093,6 +1108,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
self.cuda_graph_runner_for_draft_extend,
return_hidden_states_before_norm=False,
kv_loc_plan=kv_loc_plan,
)
if self.plan_stream:
@@ -1412,7 +1428,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
):
self.draft_worker._draft_extend_for_decode(batch, batch_output)
self.draft_worker._draft_extend_for_decode(
batch, batch_output, kv_loc_plan=verify_input.kv_loc_plan
)
if (
get_parallel().pp_size > 1
@@ -1468,7 +1486,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
return_kv_loc_plan=self._draft_worker is not None,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
@@ -1505,6 +1525,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
batch_output.logits_output.mm_input_embeds,
kv_loc_plan=kv_loc_plan,
)
return batch_output
@@ -302,7 +302,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=expanded_bs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -321,6 +320,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
def run_once():
# Record the metadata rebuild against the committed target-prefix
@@ -365,6 +365,8 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
encoder_lens=None,
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
spec_info=None,
)
with self._frozen_kv_target_view(forward_batch):
@@ -99,6 +99,7 @@ if is_npu():
fill_draft_extend_prepare_buffers = fill_draft_extend_prepare_buffers_native
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.speculative.multi_layer_eagle_worker_v2 import (
MultiLayerEagleDraftWorker,
)
@@ -333,7 +334,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Forward batch
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -361,6 +361,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
global_num_token_non_padded_cpu=self.captured_req_width * bs,
return_hidden_states_before_norm=True,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
return forward_batch
def _postprocess_forward_batch(self, forward_batch: ForwardBatch, bs: int):
@@ -476,6 +477,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
seq_lens_cpu: Optional[torch.Tensor],
*,
out_cache_loc_virtual: Optional[torch.Tensor],
kv_loc_plan: Optional[KVLocPlan],
kv_loc_cols: Optional[Cols],
):
"""Init this step's attention metadata for the prepared bucket and
replay its graph. Buffers must already be populated by the composite
@@ -501,6 +504,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
out_cache_loc_virtual=out_cache_loc_virtual,
kv_loc_plan=kv_loc_plan,
kv_loc_cols=kv_loc_cols,
spec_info=spec_info,
)
if (
@@ -550,6 +555,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
self.num_front_tokens = 0
self.prune_draft_extend_logits = False
self._out_cache_loc_virtual = None
self._kv_loc_plan = None
self._kv_loc_cols = None
self._init_and_capture()
@@ -740,11 +747,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
req_pool_indices,
out_cache_loc,
positions=None,
out_cache_loc_virtual=None,
out_cache_loc_virtual,
kv_loc_plan,
):
# Staging runs before `prepare`, so the rail arrives here rather than
# off a ForwardBatch; `prepare` overwrites it with the batch's own.
# Staging runs before `prepare` has a ForwardBatch, so the virtual write
# ids and the plan come in here; `prepare` replaces them with the batch's.
self._out_cache_loc_virtual = out_cache_loc_virtual
self._kv_loc_plan = kv_loc_plan
self._kv_loc_cols = None
raw_bs = req_pool_indices.shape[0]
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
buffers = self.buffers
@@ -780,6 +790,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
out_cache_loc_virtual=self._out_cache_loc_virtual,
kv_loc_plan=self._kv_loc_plan,
kv_loc_cols=self._kv_loc_cols,
)
for backend in backends:
backend.init_forward_metadata_out_graph(batch)
@@ -792,6 +804,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
num_tokens = raw_bs * self.captured_req_width
self._out_cache_loc_virtual = forward_batch.out_cache_loc_virtual
self._kv_loc_plan = forward_batch.kv_loc_plan
self._kv_loc_cols = forward_batch.kv_loc_cols
# Bucketize to a captured batch size (padding the tail).
if self.require_mlp_tp_gather:
@@ -900,6 +914,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
self._replay_spec_info,
self.seq_lens_cpu,
out_cache_loc_virtual=self._out_cache_loc_virtual,
kv_loc_plan=self._kv_loc_plan,
kv_loc_cols=self._kv_loc_cols,
)
raw_bs = self.raw_bs
raw_num_tokens = self.raw_num_tokens
@@ -17,7 +17,7 @@ from __future__ import annotations
import logging
import time
from dataclasses import replace
from typing import TYPE_CHECKING, List, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
@@ -107,6 +107,7 @@ _is_cpu = is_cpu()
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner, ModelRunnerOutput
@@ -460,7 +461,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
def draft(self, batch: ScheduleBatch):
draft_input: EagleDraftInput = batch.spec_info
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
draft_input,
self.req_to_token_pool,
batch,
@@ -468,12 +469,14 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[0],
self.topk,
self.speculative_num_steps,
target_translator=self.target_worker.model_runner.kv_index_translator,
num_draft_tokens=self.speculative_num_draft_tokens,
)
# Run draft
parent_list, top_scores_index, draft_tokens = self.draft_forward(forward_batch)
return build_eagle_verify_input(
verify_input = build_eagle_verify_input(
batch,
draft_input,
parent_list,
@@ -487,6 +490,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
tree_mask_mode=self.tree_mask_mode,
device=self.device,
)
if kv_loc_plan is not None:
# Verify and draft extend write the window the draft planned.
verify_input.kv_loc_plan = kv_loc_plan
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
return verify_input
def draft_forward(self, forward_batch: ForwardBatch):
# Parse args
@@ -596,6 +604,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch: ScheduleBatch,
target_hidden_states: torch.Tensor,
next_token_ids: torch.Tensor,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""
Run draft model extend to correctly fill the KV cache.
@@ -604,6 +613,8 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch: The batch to run.
target_hidden_states: Hidden states from the target model forward
next_token_ids: Next token ids generated from the target forward.
kv_loc_plan: The target prefill's plan; the draft writes the same
slots.
"""
# The draft embed clamps unconditionally (to tolerate multimodal pad
# sentinels), so probe next_token_ids here first -- otherwise a corrupted id
@@ -641,6 +652,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[0],
capture_hidden_mode=draft_capture_hidden_mode,
return_hidden_states_before_norm=True,
kv_loc_plan=kv_loc_plan,
)
self._apply_deferred_mamba_init_to_draft_pools(forward_batch)
@@ -766,14 +778,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
draft_token_num=self.speculative_num_draft_tokens,
device=batch.device,
)
seq_lens = batch.seq_lens + self.speculative_num_draft_tokens
runner.stage_shared_reads(
seq_lens=batch.seq_lens + self.speculative_num_draft_tokens,
seq_lens=seq_lens,
req_pool_indices=batch.req_pool_indices,
out_cache_loc=locs,
positions=positions,
# Both loc sources read req_to_token untranslated, so `locs` is the
# PRE-translate write loc the capture rail needs.
# Both loc sources read req_to_token untranslated: `locs` is virtual.
out_cache_loc_virtual=locs,
kv_loc_plan=self.draft_runner_list[0].kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=None,
write_virtual=locs,
),
)
return True
@@ -783,6 +801,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch_result: GenerationBatchResult,
*,
staged: bool = False,
kv_loc_plan: Optional[KVLocPlan] = None,
):
# Batch 2: Draft extend
draft_extend_input = EagleDraftExtendInput(
@@ -826,6 +845,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
return_hidden_states_before_norm=True,
widened_out_cache_loc=boundary_kv_locs,
widened_positions=boundary_kv_positions,
kv_loc_plan=kv_loc_plan,
)
if self.plan_stream:
@@ -1091,7 +1111,9 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
return_kv_loc_plan=True,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
@@ -1106,6 +1128,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
batch,
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
kv_loc_plan=kv_loc_plan,
)
return batch_output
else:
@@ -1134,7 +1157,10 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
if on_publish is not None:
on_publish(batch_output.new_seq_lens)
self.draft_worker._draft_extend_for_decode(
batch, batch_output, staged=staged
batch,
batch_output,
staged=staged,
kv_loc_plan=verify_input.kv_loc_plan,
)
return batch_output
+9 -7
View File
@@ -5,6 +5,7 @@ from typing import List, Optional
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -63,11 +64,12 @@ class NgramVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
bs = req_pool_indices.numel()
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
@@ -83,13 +85,13 @@ class NgramVerifyInput(SpecInput):
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.
@@ -297,7 +297,7 @@ class UnoWorkerV2(BaseSpecWorker):
draft_token_num=self.forward_width,
)
return ForwardBatch(
forward_batch = ForwardBatch(
forward_mode=ForwardMode.TARGET_VERIFY,
batch_size=len(prefix_lens),
input_ids=input_ids,
@@ -312,6 +312,8 @@ class UnoWorkerV2(BaseSpecWorker):
capture_hidden_mode=CaptureHiddenMode.NULL,
return_hidden_states_before_norm=False,
)
self.model_runner.kv_index_translator.bind_own_plan(forward_batch)
return forward_batch
def _run_target_block(
self,
@@ -761,6 +761,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
batch.extend_prefix_lens = torch.tensor(
@@ -26,6 +26,7 @@ from sglang.srt.environ import envs
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
@@ -419,6 +420,13 @@ class MockDSV4ModelRunner:
identity = torch.arange(swa_size, dtype=torch.int64, device=device)
self.token_to_kv_pool.register_mapping(identity)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.kv_index_translator = KVIndexTranslator(
req_to_token=self.req_to_token_pool.req_to_token,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
token_to_kv_pool=self.token_to_kv_pool,
page_size=case.page_size,
device=device,
)
self.attention_chunk_size = None
self.hisparse_coordinator = None
self.init_new_workspace = False
@@ -661,6 +669,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
# extend_* fields are only populated for extend-shaped modes. DECODE leaves
# them at their defaults; the flash_mla path reads metadata directly from
# DSV4AttnMetadata so the extend fields are unused for the compress_ratio=0
@@ -322,6 +322,7 @@ class MockGDNModelRunner(ModelRunner):
page_size=case.page_size,
get_kvcache=lambda: self.token_to_kv_pool,
)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -329,7 +330,6 @@ class MockGDNModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -562,6 +562,8 @@ def _make_forward_batch(
else None
),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -318,6 +318,7 @@ class MockKDAModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -325,7 +326,6 @@ class MockKDAModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -552,6 +552,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -327,6 +327,7 @@ class MockLightningModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -334,7 +335,6 @@ class MockLightningModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -523,6 +523,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -460,7 +461,6 @@ class MockMamba2ModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -653,6 +653,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -307,6 +307,7 @@ class MockMLAModelRunner(ModelRunner):
enable_memory_saver=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -314,7 +315,6 @@ class MockMLAModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = True
self.is_draft_worker = False
self._kernel_warmed_up = True
# Runner-mode helpers mutate speculative graph sizes after construction.
self.graph_shared_output = GraphSharedOutput(
@@ -693,6 +693,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -297,6 +297,8 @@ def _init_cuda_graph_replay_metadata(backend, capture_batch_size: int, batch):
encoder_lens=batch.encoder_lens,
extend_seq_lens=getattr(batch, "extend_seq_lens", None),
out_cache_loc=getattr(batch, "out_cache_loc", None),
kv_loc_plan=batch.kv_loc_plan,
kv_loc_cols=batch.kv_loc_cols,
spec_info=batch.spec_info,
)
backend.init_forward_metadata_out_graph(fb_view)
@@ -510,6 +510,7 @@ def run_eagle_draft_cuda_graph_runner_case(
)
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
expected = _run_eagle_draft_eager(
eager_worker,
eager_batch,
@@ -526,6 +527,7 @@ def run_eagle_draft_cuda_graph_runner_case(
)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
graph_runner = _capture_eagle_draft_graph_runner(
graph_worker,
graph_backend,
@@ -565,6 +567,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
)
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
expected = _run_frozen_kv_mtp_eager(eager_worker, eager_batch)
graph_fixture, graph_worker, _ = _build_frozen_kv_mtp_fixture(
@@ -576,6 +579,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
graph_runner = _capture_frozen_kv_mtp_graph_runner(graph_worker)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)