[Refactor] Translate each KV loc once per iteration (#42753)
This commit is contained in:
@@ -30,10 +30,11 @@ Two delivery forms over that one formula:
|
||||
which the pool bounds, where a page table's width is bounded
|
||||
only by `max_context_len`.
|
||||
|
||||
PREFIX-ONLY per row: nothing past the row's live prefix is written, so a
|
||||
caller-owned buffer keeps what it had there -- which is what lets a captured
|
||||
cuda-graph buffer be refreshed in place. Readers bound themselves by
|
||||
`cache_seqlens` and never look past the prefix.
|
||||
PREFIX-ONLY per row by default: nothing past the row's live prefix is written,
|
||||
so a caller-owned buffer keeps what it had there -- which is what lets a
|
||||
captured cuda-graph buffer be refreshed in place. Readers bound themselves by
|
||||
`cache_seqlens` and never look past the prefix. A fresh (`torch.empty`) table
|
||||
asks for the sink past the prefix instead (`zero_tail`).
|
||||
|
||||
A `-1` in `req_to_token` and a freed (`-1`) v2p row both clamp to entry 0, the
|
||||
reserved padding slot, so a kernel dereferences padding, not a wild address.
|
||||
@@ -60,6 +61,56 @@ _TARGET_BLOCKS = 1024
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_translate_items(
|
||||
row_in, # this row of `req_to_token` -- VIRTUAL token ids
|
||||
row_out, # where this row's items land
|
||||
v2p_ptr, # [num_pages + 1] int64 -- virtual->physical page table
|
||||
live, # scalar: the row reads `req_to_token`; else every item is the sink
|
||||
kv_start, # first token of the row's window
|
||||
n_items, # live items in the row
|
||||
n_written, # items stored: `n_items`, or a wider zeroed row
|
||||
item_begin, # this program's first item
|
||||
item_stride, # items one program advances per loop trip
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
EMIT_PER_TOKEN: tl.constexpr,
|
||||
OUT_INT64: tl.constexpr,
|
||||
ZERO_TAIL: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
for start in range(item_begin, n_written, item_stride):
|
||||
item = start + tl.arange(0, BLOCK)
|
||||
mask = item < n_items
|
||||
load_mask = mask & live
|
||||
pos = kv_start + item
|
||||
if EMIT_PER_TOKEN:
|
||||
page = pos // PAGE_SIZE
|
||||
else:
|
||||
page = pos
|
||||
tok = tl.load(
|
||||
row_in + page.to(tl.int64) * PAGE_SIZE, mask=load_mask, other=0
|
||||
).to(tl.int64)
|
||||
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
|
||||
# -1 at ps == 1, which would read one element BEFORE `v2p`.
|
||||
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
|
||||
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=load_mask, other=0), 0)
|
||||
if EMIT_PER_TOKEN:
|
||||
value = entry * PAGE_SIZE + pos % PAGE_SIZE
|
||||
else:
|
||||
value = entry
|
||||
store_mask = mask
|
||||
if ZERO_TAIL:
|
||||
value = tl.where(mask, value, 0)
|
||||
store_mask = item < n_written
|
||||
if OUT_INT64:
|
||||
tl.store(row_out + item, value, mask=store_mask)
|
||||
else:
|
||||
tl.store(row_out + item, value.to(tl.int32), mask=store_mask)
|
||||
|
||||
|
||||
# The per-call widths and lengths stay unspecialized: Triton would otherwise
|
||||
# compile a new variant (`== 1`, `% 16`) mid-serving each time a batch's shape
|
||||
# hits a combination it has not seen.
|
||||
@triton.jit(do_not_specialize=["out_stride", "seq_len_delta", "max_row_items"])
|
||||
def build_kv_read_indices_kernel(
|
||||
req_to_token_ptr, # in: [max_reqs, max_context] -- VIRTUAL token ids
|
||||
req_pool_indices_ptr, # in: [bs] -- req_to_token row per batch lane
|
||||
@@ -76,6 +127,7 @@ def build_kv_read_indices_kernel(
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
EMIT_PER_TOKEN: tl.constexpr,
|
||||
OUT_INT64: tl.constexpr,
|
||||
ZERO_TAIL: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
@@ -98,30 +150,24 @@ def build_kv_read_indices_kernel(
|
||||
# A widened window can outrun a fixed-width row (a verify near the
|
||||
# context limit); writing past it would land in the next row.
|
||||
n_items = tl.minimum(n_items, max_row_items)
|
||||
|
||||
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
|
||||
item = start + tl.arange(0, BLOCK)
|
||||
mask = item < n_items
|
||||
pos = kv_start + item
|
||||
if EMIT_PER_TOKEN:
|
||||
page = pos // PAGE_SIZE
|
||||
else:
|
||||
page = pos
|
||||
tok = tl.load(row_in + page.to(tl.int64) * PAGE_SIZE, mask=mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
|
||||
# -1 at ps == 1, which would read one element BEFORE `v2p`.
|
||||
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
|
||||
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=mask, other=0), 0)
|
||||
if EMIT_PER_TOKEN:
|
||||
value = entry * PAGE_SIZE + pos % PAGE_SIZE
|
||||
else:
|
||||
value = entry
|
||||
if OUT_INT64:
|
||||
tl.store(row_out + item, value, mask=mask)
|
||||
else:
|
||||
tl.store(row_out + item, value.to(tl.int32), mask=mask)
|
||||
# A fresh (unzeroed) table also takes the sink past the live prefix.
|
||||
n_written = max_row_items if ZERO_TAIL else n_items
|
||||
_gather_translate_items(
|
||||
row_in,
|
||||
row_out,
|
||||
v2p_ptr,
|
||||
True,
|
||||
kv_start,
|
||||
n_items,
|
||||
n_written,
|
||||
tl.program_id(1) * BLOCK,
|
||||
item_stride,
|
||||
PAGE_SIZE=PAGE_SIZE,
|
||||
EMIT_PER_TOKEN=EMIT_PER_TOKEN,
|
||||
OUT_INT64=OUT_INT64,
|
||||
ZERO_TAIL=ZERO_TAIL,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
|
||||
|
||||
def _launch(
|
||||
@@ -138,6 +184,7 @@ def _launch(
|
||||
kv_start_idx: Optional[torch.Tensor],
|
||||
emit_per_token: bool,
|
||||
seq_len_delta: int = 0,
|
||||
zero_tail: bool = False,
|
||||
) -> None:
|
||||
bs = int(req_pool_indices.numel())
|
||||
item_programs = min(
|
||||
@@ -159,6 +206,7 @@ def _launch(
|
||||
PAGE_SIZE=page_size,
|
||||
EMIT_PER_TOKEN=emit_per_token,
|
||||
OUT_INT64=out.dtype == torch.int64,
|
||||
ZERO_TAIL=zero_tail,
|
||||
BLOCK=_BLOCK_ITEMS,
|
||||
num_warps=_NUM_WARPS,
|
||||
)
|
||||
@@ -188,17 +236,18 @@ def build_kv_read_table(
|
||||
max_pages: int,
|
||||
out: torch.Tensor,
|
||||
seq_len_delta: int = 0,
|
||||
zero_tail: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Fill ``out``'s live prefix with PAGE TABLE entries.
|
||||
|
||||
``out`` is caller-owned (fresh zeros for the eager path, the module's
|
||||
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
|
||||
region's live prefix is written -- never rebound, never tail-cleared.
|
||||
``out`` is caller-owned and only its ``[:bs, :max_pages]`` region is
|
||||
written: each row's live prefix, and with ``zero_tail`` the sink (0) past
|
||||
it up to ``max_pages`` -- what a fresh ``torch.empty`` table needs, in the
|
||||
same launch. Never rebound.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). Every row stops at
|
||||
``max_pages``: a widened window that outruns the row is cut there rather
|
||||
than spilling into the next row, on CUDA as on CPU.
|
||||
``seq_len_delta`` widens every row's live prefix (a verify reads its draft
|
||||
tokens' KV back from the pool); a widened row is cut at ``max_pages``
|
||||
rather than spilling into the next row.
|
||||
"""
|
||||
bs = int(req_pool_indices.numel())
|
||||
assert out.dtype == torch.int32, (
|
||||
@@ -228,6 +277,8 @@ def build_kv_read_table(
|
||||
v2p=v2p,
|
||||
page_size=page_size,
|
||||
).to(torch.int32)
|
||||
if zero_tail:
|
||||
out[b, live:max_pages] = 0
|
||||
return out
|
||||
|
||||
_launch(
|
||||
@@ -243,6 +294,7 @@ def build_kv_read_table(
|
||||
kv_start_idx=None,
|
||||
emit_per_token=False,
|
||||
seq_len_delta=seq_len_delta,
|
||||
zero_tail=zero_tail,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -89,6 +89,29 @@ def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
|
||||
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_token_ids(
|
||||
row_ptr,
|
||||
pos,
|
||||
mask,
|
||||
v2p,
|
||||
page_size: tl.constexpr,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr,
|
||||
TRANSLATE: tl.constexpr,
|
||||
):
|
||||
"""Token ids at positions ``pos`` of one read-source row: the row's own
|
||||
entries when token-granular, ``entry * ps + pos % ps`` when page-granular;
|
||||
with ``TRANSLATE``, virtual ids mapped through ``v2p`` on the way out."""
|
||||
if ENTRY_PAGE_SIZE == 1:
|
||||
ids = tl.load(row_ptr + pos, mask=mask)
|
||||
else:
|
||||
entry = tl.load(row_ptr + pos // ENTRY_PAGE_SIZE, mask=mask, other=0)
|
||||
ids = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
|
||||
if TRANSLATE:
|
||||
ids = _translate_token_ids(ids, v2p, mask, page_size)
|
||||
return ids
|
||||
|
||||
|
||||
@triton.jit
|
||||
def generate_draft_decode_kv_indices(
|
||||
req_pool_indices,
|
||||
@@ -97,8 +120,9 @@ def generate_draft_decode_kv_indices(
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
positions,
|
||||
v2p,
|
||||
pool_len: tl.constexpr,
|
||||
# Runtime, not constexpr: the unified pool's read table is as wide as the
|
||||
# iteration's longest row, so a constexpr stride would recompile per width.
|
||||
row_stride,
|
||||
kv_indices_stride: tl.constexpr,
|
||||
kv_indptr_stride: tl.constexpr,
|
||||
bs_upper: tl.constexpr,
|
||||
@@ -108,6 +132,9 @@ def generate_draft_decode_kv_indices(
|
||||
window_size: tl.constexpr = 0,
|
||||
sink_size: tl.constexpr = 0,
|
||||
NUM_STEPS: tl.constexpr = 0,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr = 1,
|
||||
# The source's page table (`KVIndexTable.v2p`), applied with TRANSLATE.
|
||||
v2p=None,
|
||||
TRANSLATE: tl.constexpr = False,
|
||||
):
|
||||
# window_size > 0 restricts the draft (not the target) to sink_size prefix
|
||||
@@ -151,7 +178,7 @@ def generate_draft_decode_kv_indices(
|
||||
# Update kv_indices
|
||||
kv_offset = cum_seq_len * topk + bid * iters * topk + topk_id * (seq_len_w + iters)
|
||||
kv_ptr = kv_indices + kv_offset
|
||||
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * pool_len
|
||||
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
|
||||
|
||||
num_loop = tl.cdiv(seq_len_w, BLOCK_SIZE)
|
||||
if NUM_STEPS != 0 and blk >= num_loop and blk > 0:
|
||||
@@ -165,9 +192,9 @@ def generate_draft_decode_kv_indices(
|
||||
copy_offset,
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
data = _load_token_ids(
|
||||
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
|
||||
)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
copy_offset += BLOCK_SIZE
|
||||
else:
|
||||
@@ -179,9 +206,9 @@ def generate_draft_decode_kv_indices(
|
||||
copy_offset,
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
data = _load_token_ids(
|
||||
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
|
||||
)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
|
||||
# Extension entries and kv_indptr belong to token block 0 alone; other
|
||||
@@ -189,12 +216,14 @@ def generate_draft_decode_kv_indices(
|
||||
if blk == 0:
|
||||
extend_offset = tl.arange(0, iter_upper)
|
||||
if page_size == 1 or topk == 1:
|
||||
extend_data = tl.load(
|
||||
token_pool_ptr
|
||||
+ seq_len
|
||||
+ topk_id * num_steps
|
||||
+ tl.arange(0, iter_upper),
|
||||
mask=extend_offset < iters,
|
||||
extend_data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
seq_len + topk_id * num_steps + tl.arange(0, iter_upper),
|
||||
extend_offset < iters,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
else:
|
||||
prefix_len = seq_len
|
||||
@@ -208,14 +237,14 @@ def generate_draft_decode_kv_indices(
|
||||
+ topk_id * num_new_pages_per_topk * page_size
|
||||
+ last_page_len
|
||||
)
|
||||
extend_data = tl.load(
|
||||
token_pool_ptr + start + extend_offset,
|
||||
mask=extend_offset < iters,
|
||||
)
|
||||
|
||||
if TRANSLATE:
|
||||
extend_data = _translate_token_ids(
|
||||
extend_data, v2p, extend_offset < iters, page_size
|
||||
extend_data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
start + extend_offset,
|
||||
extend_offset < iters,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
|
||||
tl.store(
|
||||
|
||||
@@ -20,6 +20,9 @@ class RaggedVerifyWindow(msgspec.Struct, frozen=True):
|
||||
positions: torch.Tensor
|
||||
verify_cache_loc: torch.Tensor
|
||||
verify_ids: torch.Tensor
|
||||
# Each packed lane's flat position in the `[bs, verify_num_draft_tokens]`
|
||||
# verify window, -1 on a padded lane.
|
||||
window_index: torch.Tensor
|
||||
|
||||
|
||||
class BuildRaggedVerifyWindow:
|
||||
@@ -129,6 +132,9 @@ def build_ragged_verify_window(
|
||||
positions=positions,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_ids=verify_ids,
|
||||
window_index=torch.where(
|
||||
valid, safe_req.to(torch.int64) * verify_num_draft_tokens + within, -1
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -176,7 +182,7 @@ def build_ragged_verify_window_triton(
|
||||
verify_lens = layout.verify_lens.to(device=device, dtype=torch.int32)
|
||||
padded_total = layout.graph_num_tokens
|
||||
|
||||
req_id, within, _valid = compact_row_index_triton(
|
||||
req_id, within, valid = compact_row_index_triton(
|
||||
verify_lens=verify_lens, padded_total=padded_total, device=device
|
||||
)
|
||||
real_cache_loc = assign_extend_cache_locs_func(
|
||||
@@ -218,6 +224,11 @@ def build_ragged_verify_window_triton(
|
||||
positions=positions,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_ids=verify_ids,
|
||||
window_index=torch.where(
|
||||
valid,
|
||||
req_id.clamp(max=bs - 1).to(torch.int64) * verify_num_draft_tokens + within,
|
||||
-1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -499,10 +499,9 @@ def _assert_spec_verify_backends(
|
||||
) -> None:
|
||||
"""Refuse target backends whose verify id rails are not translation-audited.
|
||||
|
||||
Only the target's prefill/decode pair is checked here. A draft that fuses
|
||||
into the target's pages must run on the translated rails; the fused-draft
|
||||
decision enforces that by declining fusion otherwise, and a private-pool
|
||||
draft indexes its own pool by virtual id."""
|
||||
Only the target's prefill/decode pair is checked here: fusion declines for
|
||||
a draft off the translated rails, and a private-pool draft indexes its own
|
||||
pool by virtual id."""
|
||||
if allowed is None:
|
||||
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
|
||||
dcp_note = ""
|
||||
@@ -591,8 +590,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"DFLASH",
|
||||
), (
|
||||
"--enable-unified-memory only supports --speculative-algorithm "
|
||||
"DSPARK (chain draft), DFLASH (fused block draft), and "
|
||||
"EAGLE/EAGLE3 (fused draft KV); other speculative algorithms are "
|
||||
"DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are "
|
||||
"not yet audited for the unified pool's virtual-to-physical loc "
|
||||
"translation. Got "
|
||||
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
|
||||
@@ -627,9 +625,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
f"spec-verify-audited attention backends {sorted(eagle_allowed)} "
|
||||
f"(got {sorted(eagle_backends, key=str)})."
|
||||
)
|
||||
# An unset draft backend inherits the target's pair. A fused draft is
|
||||
# MHA-shaped on every host, and these rails also serve a draft that
|
||||
# keeps an MLA pool of its own.
|
||||
# An unset draft backend inherits the target's pair.
|
||||
draft_backend = cfg.speculative_draft_attention_backend
|
||||
draft_backends = {draft_backend} if draft_backend else eagle_backends
|
||||
assert draft_backends <= TRANSLATED_MHA_RAILS, (
|
||||
@@ -649,9 +645,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
)
|
||||
assert not cfg.enable_two_batch_overlap, (
|
||||
"--enable-unified-memory does not support --enable-two-batch-overlap: "
|
||||
"TBO's replay split hands each child a view without the pre-translate "
|
||||
"write loc, so a captured decode replay raises. "
|
||||
"TODO(ch-wan): carry out_cache_loc_virtual into the child view."
|
||||
"the iteration's read tables are indexed by the whole batch's rows, and "
|
||||
"a TBO child's readers do not offset into them."
|
||||
)
|
||||
assert not cfg.enable_lmcache, (
|
||||
"--enable-unified-memory is not yet compatible with --enable-lmcache: "
|
||||
@@ -679,8 +674,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
), (
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
|
||||
"--disaggregation-decode-retraction-backup=host_pool: the backup builds "
|
||||
"the draft's host pool off a device pool of its own, and an EAGLE draft "
|
||||
"on the unified pool lives fused in the target's pages, without one."
|
||||
"the draft's host pool off the draft's own device pool, which a fused "
|
||||
"EAGLE draft does not have."
|
||||
)
|
||||
assert not (
|
||||
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
|
||||
|
||||
@@ -698,6 +698,14 @@ class TboForwardBatchPreparer:
|
||||
start_token_index:end_token_index
|
||||
]
|
||||
output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical
|
||||
# Unified memory refuses two-batch overlap, so the plan's reads stay in
|
||||
# `req_to_token`, which each child indexes at its own rows; each child
|
||||
# writes its own tokens of the plan's window.
|
||||
output_dict["kv_loc_plan"] = batch.kv_loc_plan
|
||||
if batch.kv_loc_plan is not None:
|
||||
output_dict["kv_loc_cols"] = batch.kv_loc_plan.cols_slice(
|
||||
batch.kv_loc_cols, slice(start_token_index, end_token_index)
|
||||
)
|
||||
|
||||
attention_tp_size = get_parallel().attn_tp_size
|
||||
_tbo_padded_len = (
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.srt.layers.attention.triton_backend import (
|
||||
ForwardMetadata,
|
||||
update_sliding_window_buffer,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
get_capture_attention_variant,
|
||||
@@ -202,12 +203,10 @@ class DllmFlashAttention:
|
||||
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.use_sliding_window_kv_pool
|
||||
else None,
|
||||
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.kv_index_translator.is_translating
|
||||
else None,
|
||||
)
|
||||
metadata = self._prefill_graph_metadata[key]
|
||||
backend._fill_kv_indptr_and_indices(
|
||||
batch.kv_loc_plan,
|
||||
bs,
|
||||
batch.extend_prefix_lens,
|
||||
batch.req_pool_indices,
|
||||
@@ -220,6 +219,7 @@ class DllmFlashAttention:
|
||||
_, _, _, offsets = update_sliding_window_buffer(
|
||||
metadata.window_kv_indptr,
|
||||
backend.kv_index_translator,
|
||||
batch.kv_loc_plan,
|
||||
batch.req_pool_indices,
|
||||
backend.sliding_window_size,
|
||||
batch.extend_prefix_lens,
|
||||
@@ -231,15 +231,7 @@ class DllmFlashAttention:
|
||||
metadata.window_kv_offsets.copy_(offsets)
|
||||
if metadata.swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc.copy_(
|
||||
backend.kv_index_translator.sliding_window_write_loc_for(
|
||||
batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
if metadata.out_cache_loc_full_physical is not None:
|
||||
backend.kv_index_translator.fill_capture_write_loc(
|
||||
out=metadata.out_cache_loc_full_physical,
|
||||
forward_batch=batch,
|
||||
width=metadata.out_cache_loc_full_physical.numel(),
|
||||
backend.kv_index_translator.write_ids(batch, IdSpaceKind.SLIDING_WINDOW)
|
||||
)
|
||||
backend.forward_metadata = metadata
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -1951,6 +1952,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
paged_kernel_lens=forward_batch.seq_lens,
|
||||
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
)
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -2150,6 +2152,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
max(forward_batch.extend_seq_lens_cpu),
|
||||
forward_batch.seq_lens_cpu.max().item(),
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
|
||||
max_q_len = self.mla_indices_updater_prefill.max_q_len
|
||||
@@ -2190,6 +2193,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
prefix_lens,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.use_sliding_window_kv_pool:
|
||||
@@ -4454,6 +4458,8 @@ class AiterIndicesUpdaterPrefill:
|
||||
prefix_lens: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -4466,6 +4472,8 @@ class AiterIndicesUpdaterPrefill:
|
||||
prefix_lens: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
kv_start_idx = None
|
||||
kv_indptr = self.kv_indptr
|
||||
@@ -4515,6 +4523,7 @@ class AiterIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4547,6 +4556,8 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
max_q_len: int,
|
||||
max_kv_len: int,
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -4560,6 +4571,8 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
max_q_len: int,
|
||||
max_kv_len: int,
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
bs = len(req_pool_indices)
|
||||
|
||||
@@ -4594,6 +4607,7 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=kv_lens,
|
||||
paged_kernel_lens_sum=kv_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4663,18 +4677,21 @@ class AiterMultiStepDraftBackend:
|
||||
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
|
||||
else 1
|
||||
)
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
triton.next_power_of_2(num_seqs),
|
||||
@@ -4684,7 +4701,9 @@ class AiterMultiStepDraftBackend:
|
||||
# A single token block is the historical launch; NUM_STEPS=0 keeps
|
||||
# its 128-wide program instead of the token-block specialization.
|
||||
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_flashinfer_available
|
||||
|
||||
@@ -176,7 +177,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
query = self._set_kv_and_concat_q_fp8_fused(
|
||||
layer=layer,
|
||||
@@ -208,7 +209,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
|
||||
assert k is not None and k_rope is not None
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
self._kv_write_loc(forward_batch),
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
k,
|
||||
k_rope,
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver
|
||||
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -54,6 +54,7 @@ from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.srt.utils.common import get_device_capability
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -466,10 +467,9 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
return
|
||||
if self.kv_index_translator.is_translating:
|
||||
# The draft-extend runners record this call unconditionally, but a
|
||||
# translating backend rebuilds these tables out of graph on every
|
||||
# replay; a captured raw req_to_token gather would then overwrite
|
||||
# them with virtual ids.
|
||||
# The draft-extend runners record this call unconditionally; a
|
||||
# translating backend rebuilds these tables out of graph, and a
|
||||
# captured req_to_token gather would overwrite them with virtual ids.
|
||||
return
|
||||
bs = forward_batch.batch_size
|
||||
metadata = self.draft_extend_metadata[bs]
|
||||
@@ -543,7 +543,11 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
self._init_full_cg_decode_metadata(forward_batch, in_capture)
|
||||
|
||||
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
|
||||
locations = getattr(forward_batch, "out_cache_loc", None)
|
||||
"""This step's sliding-window write ids: the plan's, sliced per
|
||||
draft step the way `out_cache_loc` is."""
|
||||
locations = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
if (
|
||||
locations is not None
|
||||
and forward_batch.forward_mode.is_decode_or_idle()
|
||||
@@ -586,11 +590,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if in_capture:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
|
||||
|
||||
if in_capture:
|
||||
num_tokens = forward_batch.positions.numel()
|
||||
@@ -627,6 +627,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
return
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -678,6 +679,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# A stale non-None seq_lens_cpu buffer would under-size max_seq_pages
|
||||
# (stale page-table rows -> OOB); force None under sync-free.
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -755,22 +757,17 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
translating = self.kv_index_translator.is_translating
|
||||
max_seq_len_k = int(forward_batch.seq_lens_cpu[:bs].max().item())
|
||||
if translating:
|
||||
# Unified pool: the block table is a TRANSLATED page table, built
|
||||
# straight into these capture-stable buffers from the LIVE v2p, so
|
||||
# a page relocated by compaction since capture is picked up. Same
|
||||
# substitution the eager extend branch makes in its `_unified_read`
|
||||
# fixup; `build_index_table` emits page-granular physical ids
|
||||
# directly, so there is no `// page_size` to undo.
|
||||
self.kv_index_translator.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices[:bs],
|
||||
seq_lens=forward_batch.seq_lens[:bs],
|
||||
into=KVReadTables(
|
||||
full=m.page_table,
|
||||
sliding_window=(
|
||||
m.swa_page_table if self.use_sliding_window_kv_pool else None
|
||||
),
|
||||
),
|
||||
# Unified pool: the plan's page table, physical page ids from this
|
||||
# iteration's v2p, copied into these capture-stable buffers.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=m.page_table
|
||||
)
|
||||
if self.use_sliding_window_kv_pool:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
out=m.swa_page_table,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
elif max_seq_len_k > 0:
|
||||
# Build the block table like the eager extend branch: take every
|
||||
# page_size-th token slot from req_to_token and divide by page_size.
|
||||
@@ -797,22 +794,15 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
self.full_cg_prefill_swa_out_cache_loc.shape[0],
|
||||
"full-CG prefill SWA write-location buffer",
|
||||
)
|
||||
# Under the unified pool `out_cache_loc` was rebound to FULL-side
|
||||
# physical ids at ForwardBatch construction, so the full->swa map
|
||||
# cannot be re-run on it: a full physical page number is also a
|
||||
# valid virtual page of the swa v2p, so the map would silently
|
||||
# resolve it to an unrelated token's swa slot. Phase 2 of the write
|
||||
# contract derives the swa loc from them instead.
|
||||
swa_write_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
if in_capture:
|
||||
# The runner-built capture batch writes the page-0 sink.
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].zero_()
|
||||
else:
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
if translating
|
||||
else self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(swa_write_loc)
|
||||
# Captured kernels read the full bucket. Route its inactive tail to
|
||||
# SWA's zero dummy slot to prevent stale writes into live slots.
|
||||
self.full_cg_prefill_swa_out_cache_loc[num_out:].zero_()
|
||||
@@ -826,17 +816,14 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def _spec_read_seq_len_delta(
|
||||
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
|
||||
) -> int:
|
||||
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
|
||||
the translated table's fill must reach ``cache_seqlens``. Takes the
|
||||
two fields rather than a ForwardBatch: the captured build slices its
|
||||
batch and has none. Now read by both builds, so eager and replay
|
||||
widen identically.
|
||||
"""Columns past ``seq_lens`` this mode's whole-sequence read covers;
|
||||
the translated table must reach ``cache_seqlens``.
|
||||
|
||||
Zero for the prefix-only shapes: normal decode/extend, the topk>1
|
||||
split (drafts read via the expand metadata), draft-extend whose lens
|
||||
already include the window, and draft-extend's idle batch. The ragged
|
||||
verify layout's per-row lens are bounded by the draft window, so its
|
||||
upper bound rides the same delta."""
|
||||
split (drafts read via the expand metadata), draft-extend (whose lens
|
||||
already include the window) and its idle batch. The ragged verify
|
||||
layout's per-row lens are bounded by the draft window, so the same
|
||||
delta bounds it."""
|
||||
if spec_info is None:
|
||||
return 0
|
||||
if forward_mode.is_target_verify() and self.topk <= 1:
|
||||
@@ -1237,27 +1224,20 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else None
|
||||
)
|
||||
if _unified_read:
|
||||
delta = self._spec_read_seq_len_delta(
|
||||
forward_batch.forward_mode, forward_batch.spec_info
|
||||
# The plan's table covers this forward's reads, a verify's and a
|
||||
# draft decode's window included; the kernels bound each row by
|
||||
# `cache_seqlens`.
|
||||
kv_view = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
|
||||
)
|
||||
if delta:
|
||||
kv_view = self.kv_index_translator.widened_index_table(
|
||||
forward_batch, seq_len_delta=delta
|
||||
)
|
||||
else:
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
metadata.page_table = kv_view.ids
|
||||
if self.use_sliding_window_kv_pool:
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
if swa_out_cache_loc is not None:
|
||||
# The swa write loc was computed from the still-VIRTUAL
|
||||
# loc at ForwardBatch construction; re-running the
|
||||
# full->swa map on the physical loc would be garbage.
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
metadata.swa_page_table = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
rows=forward_batch.batch_size,
|
||||
).ids
|
||||
metadata.swa_out_cache_loc = swa_out_cache_loc
|
||||
elif self.use_sliding_window_kv_pool:
|
||||
# FA3 requires an int32 page_table.
|
||||
metadata.swa_page_table = (
|
||||
@@ -1265,12 +1245,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.page_table
|
||||
).to(torch.int32)
|
||||
)
|
||||
if swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
metadata.swa_out_cache_loc = swa_out_cache_loc
|
||||
|
||||
# Logical-page KV sharding: capture the batch's gather plan and swap the
|
||||
# page table to scratch rows. During a sharded extend, attention reads
|
||||
@@ -2864,6 +2839,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
|
||||
def _set_decode_page_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
metadata,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -2871,9 +2847,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
) -> None:
|
||||
"""Fill `cache_seqlens_int32`, `cu_seqlens_k` and the page table(s).
|
||||
|
||||
Under the unified pool the translator writes the page tables in place,
|
||||
so the fused kernel is left with the prefix sum alone -- one pass over
|
||||
the rows instead of a translated build plus a verbatim copy of it.
|
||||
Under the unified pool the page tables are the plan's, copied in, so
|
||||
the fused kernel is left with the prefix sum alone.
|
||||
"""
|
||||
translated = self.kv_index_translator.reads_are_translated
|
||||
normal_decode_set_metadata(
|
||||
@@ -2891,17 +2866,20 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
skip_page_table=translated,
|
||||
)
|
||||
if translated:
|
||||
# Fill to `cache_seqlens_int32`, which the kernels bound their reads
|
||||
# by: a draft decode reads `seq_len_delta` past `seq_lens`.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
sliding_window_out=metadata.swa_page_table,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=metadata.cache_seqlens_int32,
|
||||
)
|
||||
# The plan's table reaches `cache_seqlens_int32`, which the kernels
|
||||
# bound their reads by: a draft decode reads past `seq_lens`.
|
||||
bs = int(req_pool_indices.numel())
|
||||
self.kv_index_translator.copy_page_table(plan, out=metadata.page_table[:bs])
|
||||
if metadata.swa_page_table is not None:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan,
|
||||
out=metadata.swa_page_table[:bs],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -2937,6 +2915,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata
|
||||
# is normal-decode-only).
|
||||
self._set_decode_page_metadata(
|
||||
plan,
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
@@ -3041,6 +3020,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else self.max_context_len
|
||||
)
|
||||
self._set_decode_page_metadata(
|
||||
plan,
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
@@ -3092,7 +3072,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata is
|
||||
# normal-decode-only).
|
||||
self._set_decode_page_metadata(
|
||||
metadata, req_pool_indices, verify_lens, verify_delta
|
||||
plan, metadata, req_pool_indices, verify_lens, verify_delta
|
||||
)
|
||||
else:
|
||||
# When topk > 1, we need two specific target verify metadata, and then merge states
|
||||
@@ -3225,23 +3205,19 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.max_seq_len_k + self.page_size - 1
|
||||
) // self.page_size
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
# The translator writes kernel-facing page ids straight into
|
||||
# the captured tables; `sliding_window_out` fills the swa twin
|
||||
# in the same pass.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
sliding_window_out=(
|
||||
metadata.swa_page_table
|
||||
if self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
else None
|
||||
),
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_len_delta=self._spec_read_seq_len_delta(
|
||||
forward_mode, spec_info
|
||||
),
|
||||
rows = int(req_pool_indices.numel())
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=metadata.page_table[:rows]
|
||||
)
|
||||
if (
|
||||
self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
):
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan,
|
||||
out=metadata.swa_page_table[:rows],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
else:
|
||||
page_indices = self.req_to_token[
|
||||
req_pool_indices[:, None],
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
@@ -61,6 +62,7 @@ from sglang.srt.utils import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -749,6 +751,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
fixed_split_size=None,
|
||||
disable_split_kv=self.disable_cuda_graph_kv_split,
|
||||
req_pool_indices=req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_target_verify():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -761,6 +764,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -773,6 +777,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=not self.use_paged,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -785,6 +790,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_extend():
|
||||
# Plain EXTEND under full prefill CUDA graph. plan() runs
|
||||
@@ -803,6 +809,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
else:
|
||||
raise ValueError("Invalid forward mode")
|
||||
@@ -863,8 +870,8 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
if in_capture:
|
||||
@@ -955,8 +962,8 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
swa_out_cache_loc = None
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
@@ -970,6 +977,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
fixed_split_size=self.decode_split_tile_size,
|
||||
disable_split_kv=False,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = DecodeMetadata(
|
||||
self.decode_wrappers, swa_out_cache_loc=swa_out_cache_loc
|
||||
@@ -985,6 +993,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
spec_info=forward_batch.spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrappers_verify,
|
||||
@@ -1038,6 +1047,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
cross_attention_custom_mask=forward_batch.cross_attention_custom_mask,
|
||||
extend_prefix_lens_cpu=forward_batch.extend_prefix_lens_cpu,
|
||||
custom_kv_indices=self.dq_page_table,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrappers_paged,
|
||||
@@ -1581,6 +1591,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
@@ -1597,6 +1608,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
decode_wrappers = decode_wrappers or self.decode_wrappers
|
||||
@@ -1610,6 +1622,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
seq_lens_cpu,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1624,6 +1637,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
assert self.sliding_window_size is not None
|
||||
@@ -1663,6 +1677,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
use_sliding_window_kv_pool=use_sliding_window_kv_pool,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1677,6 +1692,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Cache encoder_lens on CPU to avoid GPU→CPU transfer per call
|
||||
@@ -1703,6 +1719,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
seq_lens_cpu=kv_lens_cpu,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1719,6 +1736,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
@@ -1726,6 +1744,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
|
||||
if spec_info is None or getattr(spec_info, "kv_indptr", None) is None:
|
||||
bs = len(paged_kernel_lens)
|
||||
kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0)
|
||||
@@ -1739,14 +1758,14 @@ class FlashInferIndicesUpdaterDecode:
|
||||
paged_kernel_lens_sum, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=use_swa_source,
|
||||
kind=read_kind,
|
||||
)
|
||||
else:
|
||||
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
|
||||
@@ -1868,6 +1887,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -1888,6 +1909,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if use_ragged:
|
||||
assert prefix_lens is not None
|
||||
@@ -1918,6 +1941,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
multi_item_params=multi_item_params,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
custom_kv_indices=custom_kv_indices,
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def update_sliding_window(
|
||||
@@ -1936,6 +1960,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if custom_kv_indices is not None:
|
||||
raise RuntimeError(
|
||||
@@ -2031,6 +2057,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
if (wrapper_id == 0 and not use_ragged and spec_info is None)
|
||||
else -1
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def _build_swa_prefix_custom_mask(
|
||||
@@ -2090,6 +2117,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if custom_kv_indices is not None:
|
||||
raise RuntimeError(
|
||||
@@ -2125,6 +2154,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask=(
|
||||
cross_attention_custom_mask if wrapper_id == 1 else None
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def call_begin_forward(
|
||||
@@ -2148,6 +2178,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
seq_lens_cpu: Optional[torch.Tensor] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
window_left: int = -1,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
bs = len(seq_lens)
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
@@ -2155,6 +2187,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
|
||||
if spec_info is None:
|
||||
assert prefix_lens is not None
|
||||
assert len(seq_lens) == len(req_pool_indices)
|
||||
@@ -2180,14 +2213,14 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
dtype=torch.int32,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=use_swa_source,
|
||||
kind=read_kind,
|
||||
)
|
||||
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
|
||||
qo_indptr = qo_indptr[: bs + 1]
|
||||
@@ -2202,7 +2235,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
plan=plan,
|
||||
kind=read_kind,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
)
|
||||
@@ -2213,7 +2247,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
plan=plan,
|
||||
kind=read_kind,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2392,18 +2427,21 @@ class FlashInferMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
@@ -2412,7 +2450,9 @@ class FlashInferMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
|
||||
FlashInferMlaAttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -367,6 +368,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
init_metadata_replay=False,
|
||||
spec_info=spec_info,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.decode_cuda_graph_metadata[bs] = decode_wrapper
|
||||
self.forward_metadata = DecodeMetadata(decode_wrapper)
|
||||
@@ -391,6 +393,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
raise ValueError(f"Invalid mode: {forward_mode=}")
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -408,6 +411,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
prefill_wrapper.plan = partial(fast_mla_prefill_plan, prefill_wrapper)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -425,6 +429,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
decode_wrapper=self.decode_wrapper,
|
||||
init_metadata_replay=False,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = DecodeMetadata(self.decode_wrapper)
|
||||
elif forward_batch.forward_mode.is_target_verify():
|
||||
@@ -436,6 +441,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
prefill_wrapper_paged=self.prefill_wrapper_verify,
|
||||
use_ragged=False,
|
||||
spec_info=forward_batch.spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False)
|
||||
else:
|
||||
@@ -484,6 +490,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr_cpu=kv_indptr_cpu,
|
||||
kv_len_arr_cpu=kv_len_arr_cpu,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrapper_paged, use_ragged
|
||||
@@ -531,6 +538,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -566,6 +574,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
init_metadata_replay=True,
|
||||
spec_info=spec_info,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
plan=plan,
|
||||
**self.fast_decode_kwargs,
|
||||
)
|
||||
elif forward_mode.is_target_verify():
|
||||
@@ -616,6 +625,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
if use_generic_fast_plan
|
||||
else None
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid forward mode: {forward_mode=}")
|
||||
@@ -859,6 +869,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
init_metadata_replay: bool = False,
|
||||
spec_info: Optional[SpecInput] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
**fast_decode_kwargs,
|
||||
):
|
||||
@@ -871,6 +882,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
self.kv_indptr,
|
||||
init_metadata_replay,
|
||||
spec_info,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
**fast_decode_kwargs,
|
||||
)
|
||||
@@ -885,6 +897,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
init_metadata_replay: bool = False,
|
||||
spec_info: Optional[SpecInput] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
**fast_decode_kwargs,
|
||||
):
|
||||
@@ -904,11 +917,11 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
else fast_decode_kwargs["kv_indices"]
|
||||
)
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
is_translated = translator.fill_packed_read_stream(
|
||||
is_translated = translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
)
|
||||
|
||||
@@ -1006,6 +1019,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
|
||||
fast_verify_plan_kwargs: Optional[dict] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
qo_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_len_arr_cpu: Optional[torch.Tensor] = None,
|
||||
@@ -1031,6 +1045,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
spec_info,
|
||||
attn_dcp_metadata=attn_dcp_metadata,
|
||||
fast_verify_plan_kwargs=fast_verify_plan_kwargs,
|
||||
plan=plan,
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr_cpu=kv_indptr_cpu,
|
||||
kv_len_arr_cpu=kv_len_arr_cpu,
|
||||
@@ -1052,6 +1067,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
|
||||
fast_verify_plan_kwargs: Optional[dict] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
qo_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_len_arr_cpu: Optional[torch.Tensor] = None,
|
||||
@@ -1068,11 +1084,11 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
dtype=torch.int32,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
self.attn_backend.kv_index_translator.fill_packed_read_stream(
|
||||
self.attn_backend.kv_index_translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
)
|
||||
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
|
||||
@@ -1085,6 +1101,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
plan=plan,
|
||||
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
|
||||
)
|
||||
)
|
||||
@@ -1097,6 +1114,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1241,25 +1259,30 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
next_power_of_2(self.speculative_num_steps),
|
||||
next_power_of_2(bs),
|
||||
self.page_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -125,6 +126,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
or forward_mode.is_draft_extend_v2()
|
||||
):
|
||||
self._apply_decode_target_verify_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=forward_batch.batch_size,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
@@ -165,10 +167,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -200,10 +200,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -240,10 +238,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens_k,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -328,6 +324,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _apply_decode_target_verify_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -355,10 +352,9 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=self.cuda_graph_kv_indices,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
# The plan's table reaches a verify's draft tail.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=self.cuda_graph_kv_indices[:bs]
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
|
||||
@@ -251,6 +251,7 @@ def _build_tbo_child_replay_fb_view(
|
||||
child_seq_lens_cpu = fb_view.seq_lens_cpu[seq_slice]
|
||||
parent_input_ids = getattr(fb_view, "input_ids", None)
|
||||
parent_out_cache_loc = getattr(fb_view, "out_cache_loc", None)
|
||||
plan = getattr(fb_view, "kv_loc_plan", None)
|
||||
return SimpleNamespace(
|
||||
batch_size=child_bs,
|
||||
forward_mode=fb_view.forward_mode,
|
||||
@@ -270,6 +271,15 @@ def _build_tbo_child_replay_fb_view(
|
||||
if parent_out_cache_loc is not None
|
||||
else None
|
||||
),
|
||||
# Unified memory refuses two-batch overlap, so the plan's reads stay in
|
||||
# `req_to_token`, which each child indexes at its own rows; each child
|
||||
# writes its own tokens of the plan's window.
|
||||
kv_loc_plan=plan,
|
||||
kv_loc_cols=(
|
||||
None
|
||||
if plan is None
|
||||
else plan.cols_slice(getattr(fb_view, "kv_loc_cols", None), tok_slice)
|
||||
),
|
||||
spec_info=child_spec_info,
|
||||
max_seq_len_override=fb_view.max_seq_len_override,
|
||||
)
|
||||
|
||||
@@ -404,7 +404,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
|
||||
if save_kv_cache:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
self._kv_write_loc(forward_batch),
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
k,
|
||||
k_rope,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.layers.dcp import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -68,6 +69,7 @@ if _is_cuda:
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -130,9 +132,6 @@ class ForwardMetadata:
|
||||
swa_attn_logits: Optional[torch.Tensor] = None
|
||||
# full->SWA translated out_cache_loc (SWA KV-store write target)
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None
|
||||
# PHYSICAL full-attn write target for the unified pool (eager: translated tensor;
|
||||
# cuda-graph: capture-stable buffer view). None for non-unified pools.
|
||||
out_cache_loc_full_physical: Optional[torch.Tensor] = None
|
||||
# Lean decode (persistent-grid partial-result buffers)
|
||||
lean_Mp: Optional[torch.Tensor] = None
|
||||
lean_Lp: Optional[torch.Tensor] = None
|
||||
@@ -503,6 +502,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _fill_kv_indptr_and_indices(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -512,17 +512,18 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if kv_indptr is None:
|
||||
kv_indptr = self.kv_indptr[: bs + 1]
|
||||
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
self.kv_index_translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
self.kv_index_translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens[:bs],
|
||||
indptr=kv_indptr,
|
||||
total_tokens=kv_indices.numel(),
|
||||
out=kv_indices,
|
||||
)
|
||||
return kv_indptr
|
||||
|
||||
def _update_decode_kv_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -551,7 +552,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
num_kv_splits_lens = dcp_seq_lens.clamp_min(1)
|
||||
else:
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
num_kv_splits_lens = seq_lens
|
||||
window_kv_indptr = self.window_kv_indptr
|
||||
@@ -560,6 +561,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_indptr, _, window_kv_lens, _ = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
seq_lens,
|
||||
@@ -577,6 +579,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_target_verify_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
spec_info,
|
||||
@@ -593,7 +596,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
window_kv_indptr = self.window_kv_indptr
|
||||
window_kv_indices = None
|
||||
@@ -607,6 +610,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
seq_lens[:bs],
|
||||
@@ -638,6 +642,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_dllm_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -654,12 +659,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
self._fill_kv_indptr_and_indices(
|
||||
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
if self.sliding_window_size is not None and self.sliding_window_size > 0:
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
prefix_lens,
|
||||
@@ -670,6 +676,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_draft_extend_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
@@ -705,7 +712,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
extend_seq_lens = torch.zeros(bs, dtype=torch.int32, device=seq_lens.device)
|
||||
kv_lens = torch.clamp(seq_lens - extend_seq_lens, min=0).to(torch.int32)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
return qo_indptr, kv_indptr, num_tokens_per_req
|
||||
|
||||
@@ -755,15 +762,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
return
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
out_cache_loc_full_physical = self._fill_cuda_graph_write_locs(
|
||||
forward_batch, bs
|
||||
)
|
||||
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
@@ -772,25 +777,25 @@ class TritonAttnBackend(AttentionBackend):
|
||||
forward_mode,
|
||||
spec_info,
|
||||
swa_out_cache_loc,
|
||||
out_cache_loc_full_physical,
|
||||
)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
# Metadata view is reused from capture; just refill the buffers.
|
||||
self._fill_cuda_graph_write_locs(forward_batch, bs)
|
||||
# Metadata view is reused from capture; the write loc is the
|
||||
# runner's slot, so only the swa twin needs refilling.
|
||||
self._fill_cuda_graph_swa_out_cache_loc(forward_batch)
|
||||
|
||||
def _fill_cuda_graph_swa_out_cache_loc(
|
||||
self, forward_batch: ForwardBatch, in_capture: bool = False
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Refill the SWA write-target buffer from the batch's derived
|
||||
sliding-window write loc, returning the [:n] view (None for non-SWA /
|
||||
"""Refill the SWA write-target buffer with the batch's write ids in the
|
||||
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
|
||||
multi-step draft) so the captured store reads fresh slots on replay.
|
||||
"""
|
||||
if not self.use_sliding_window_kv_pool:
|
||||
@@ -807,24 +812,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
return self.cuda_graph_swa_out_cache_loc[:n]
|
||||
|
||||
def _fill_cuda_graph_write_locs(
|
||||
self, forward_batch: ForwardBatch, bs: int
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Runs BEFORE graph.replay(), so it reads the live post-compaction
|
||||
v2p; no-op for non-unified pools."""
|
||||
# The buffer exists only for a translating pool; return before naming it.
|
||||
if not self.kv_index_translator.is_translating:
|
||||
return None
|
||||
return self.kv_index_translator.fill_capture_write_loc(
|
||||
out=self.cuda_graph_out_cache_loc_full_physical,
|
||||
forward_batch=forward_batch,
|
||||
width=self.cuda_graph_out_cache_loc_full_physical.numel(),
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Init auxiliary variables for triton attention backend."""
|
||||
|
||||
@@ -862,6 +855,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
seq_lens_sum, dtype=torch.int64, device=self.device
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -875,6 +869,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.seq_lens,
|
||||
@@ -961,6 +956,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
seq_lens_sum, dtype=torch.int64, device=self.device
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -977,6 +973,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
) = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.seq_lens,
|
||||
@@ -1016,6 +1013,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.extend_prefix_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -1030,6 +1028,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
) = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.extend_prefix_lens,
|
||||
@@ -1054,8 +1053,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
swa_out_cache_loc = None
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -1074,11 +1073,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets,
|
||||
swa_attn_logits=swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=(
|
||||
forward_batch.out_cache_loc
|
||||
if self.kv_index_translator.is_translating
|
||||
else None
|
||||
),
|
||||
lean_Mp=lean_Mp,
|
||||
lean_Lp=lean_Lp,
|
||||
lean_Op=lean_Op,
|
||||
@@ -1201,22 +1195,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.kv_index_translator.is_translating:
|
||||
# Unified pool full-attention write-target buffer, refilled at replay
|
||||
# (-> KVWriteLoc.full_loc). Capture-stable, mirrors cuda_graph_swa_out_cache_loc.
|
||||
self.cuda_graph_out_cache_loc_full_physical = torch.zeros(
|
||||
(max_num_tokens,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _build_cuda_graph_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[SpecInput],
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None,
|
||||
out_cache_loc_full_physical: Optional[torch.Tensor] = None,
|
||||
) -> ForwardMetadata:
|
||||
"""Construct ForwardMetadata from the current cuda-graph buffer state.
|
||||
|
||||
@@ -1247,7 +1231,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets=None,
|
||||
swa_attn_logits=self.cuda_graph_swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
lean_Mp=self.cuda_graph_lean_Mp,
|
||||
lean_Lp=self.cuda_graph_lean_Lp,
|
||||
lean_Op=self.cuda_graph_lean_Op,
|
||||
@@ -1279,7 +1262,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
),
|
||||
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
return ForwardMetadata(
|
||||
@@ -1297,7 +1279,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
return ForwardMetadata(
|
||||
@@ -1323,13 +1304,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid forward mode: {forward_mode=} for CUDA Graph.")
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -1344,7 +1325,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if forward_mode.is_decode_or_idle():
|
||||
assert spec_info is None, "Multi-step cuda graph init is not done here."
|
||||
_, _, window_kv_lens, num_kv_splits_lens = self._update_decode_kv_buffers(
|
||||
bs, seq_lens, req_pool_indices
|
||||
plan, bs, seq_lens, req_pool_indices
|
||||
)
|
||||
self.get_num_kv_splits(
|
||||
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
|
||||
@@ -1356,13 +1337,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
elif forward_mode.is_target_verify():
|
||||
bs = len(req_pool_indices)
|
||||
self._update_target_verify_buffers(
|
||||
bs, seq_lens, spec_info, req_pool_indices
|
||||
plan, bs, seq_lens, spec_info, req_pool_indices
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
|
||||
self._update_dllm_buffers(plan, bs, seq_lens, req_pool_indices)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
self._update_draft_extend_buffers(
|
||||
bs, seq_lens, forward_mode, spec_info, req_pool_indices
|
||||
plan, bs, seq_lens, forward_mode, spec_info, req_pool_indices
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -1649,7 +1630,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
loc_info = KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
)
|
||||
if layer.k_scale is None:
|
||||
self._set_kv_buffer(forward_batch, layer, loc_info, k, v)
|
||||
@@ -2123,8 +2103,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
"window-layer extend before the metadata carried a "
|
||||
"sliding-window write loc"
|
||||
)
|
||||
elif self.forward_metadata.out_cache_loc_full_physical is not None:
|
||||
extend_kv_indices = self.forward_metadata.out_cache_loc_full_physical
|
||||
|
||||
# Capture batches may not have a spec_info, so use the attention
|
||||
# metadata's resolved uniform verify width when extend lengths are absent.
|
||||
@@ -2234,14 +2212,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
k.div_(layer.k_scale)
|
||||
self.token_to_kv_pool.set_kv_buffer(
|
||||
layer,
|
||||
# `full_loc` carries the pre-translated loc under the unified
|
||||
# pool, refreshed into a capture-stable buffer before replay —
|
||||
# translating inside set_kv_buffer would be captured and replay
|
||||
# a stale v2p. None (-> raw loc) for static pools.
|
||||
KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
),
|
||||
k,
|
||||
v,
|
||||
@@ -2253,7 +2226,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
),
|
||||
k,
|
||||
v,
|
||||
@@ -2468,18 +2440,21 @@ class TritonMultiStepDraftBackend:
|
||||
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
|
||||
seq_lens_sum = num_seqs * self.max_context_len
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
@@ -2488,7 +2463,9 @@ class TritonMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
if call_fn is None:
|
||||
@@ -2590,6 +2567,7 @@ class TritonMultiStepDraftBackend:
|
||||
def update_sliding_window_buffer(
|
||||
window_kv_indptr,
|
||||
translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
sliding_window_size,
|
||||
seq_lens,
|
||||
@@ -2626,16 +2604,18 @@ def update_sliding_window_buffer(
|
||||
if isinstance(token_to_kv_pool, SWAKVPool)
|
||||
else None
|
||||
)
|
||||
translated = translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
translated = translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=window_kv_lens,
|
||||
indptr=window_kv_indptr,
|
||||
total_tokens=window_kv_indices.numel(),
|
||||
out=window_kv_indices,
|
||||
kv_start_idx=window_kv_start_idx,
|
||||
sliding_window=(
|
||||
translator.reads_are_translated
|
||||
kind=(
|
||||
IdSpaceKind.SLIDING_WINDOW
|
||||
if translator.reads_are_translated
|
||||
and isinstance(token_to_kv_pool, BaseSWAKVPool)
|
||||
else IdSpaceKind.FULL
|
||||
),
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
KVCacheAttentionAccessKind,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -1088,16 +1089,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
# fused kernel skips its page-table writes so the graph reads the
|
||||
# refreshed content through pointers baked at capture).
|
||||
metadata = self.forward_metadata
|
||||
# `cache_seqlens_int32` is what the attention kernels bound their
|
||||
# page-table reads by, and the fused metadata call above wrote it.
|
||||
# A target verify reads `draft_token_num` further than `seq_lens`
|
||||
# goes, so filling to `seq_lens` leaves those columns untranslated.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
req_pool_indices=forward_batch.req_pool_indices[:bs],
|
||||
seq_lens=metadata.cache_seqlens_int32,
|
||||
sliding_window_out=metadata.swa_page_table,
|
||||
# The kernels bound their page-table reads by `cache_seqlens_int32`
|
||||
# (the fused metadata call above wrote it); the plan's table
|
||||
# reaches that far, a target verify's draft tail included.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=metadata.page_table[:bs]
|
||||
)
|
||||
if metadata.swa_page_table is not None:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
out=metadata.swa_page_table[:bs],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
# A capture batch carries no prepared write loc; zeros are the
|
||||
# page-0 sink.
|
||||
if (
|
||||
@@ -1110,8 +1113,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1153,7 +1156,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
metadata, req_pool_indices, metadata.cache_seqlens_int32
|
||||
)
|
||||
# The fused in-graph kernel also skips ragged batches, so refill the
|
||||
# SWA write-target buffer here (out_cache_loc -> SWA locs).
|
||||
# SWA write-target buffer here, from the plan.
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
n = forward_batch.out_cache_loc.shape[0]
|
||||
self.cuda_graph_swa_out_cache_loc[n:].zero_()
|
||||
@@ -1161,8 +1164,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1266,12 +1269,23 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
metadata.cu_seqlens_q = metadata.cu_seqlens_k
|
||||
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
kv_view = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
|
||||
)
|
||||
if kv_view.is_translated:
|
||||
# No fill kernel: the kernels take the tensor's own width/stride
|
||||
# and bound their reads by cache_seqlens.
|
||||
metadata.page_table = kv_view.ids
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
metadata.swa_page_table = (
|
||||
self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
rows=forward_batch.batch_size,
|
||||
).ids
|
||||
if self.kv_index_translator.space(IdSpaceKind.SLIDING_WINDOW)
|
||||
is not None
|
||||
else None
|
||||
)
|
||||
else:
|
||||
has_swa = self._swa_kv_pool is not None
|
||||
metadata.page_table = torch.empty(
|
||||
@@ -1300,10 +1314,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
|
||||
# int64 scatter index (unlike the int32 read page table above).
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
metadata.swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
self.forward_metadata = metadata
|
||||
|
||||
@@ -79,6 +79,7 @@ if is_flashinfer_available():
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -299,10 +300,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
# Tree-mask scratch is fetched from the target backend only.
|
||||
self.is_draft_runner = model_runner.is_draft_worker
|
||||
|
||||
# [:n] view of a capture-stable buffer on the cuda-graph path; None on
|
||||
# the eager path, which passes forward_batch.out_cache_loc through.
|
||||
self._decode_kernel_loc: Optional[torch.Tensor] = None
|
||||
self.cuda_graph_out_cache_loc_kernel: Optional[torch.Tensor] = None
|
||||
# Fused KV-scatter + q-concat on the decode dense-loc path (one launch
|
||||
# instead of set_mla_kv_buffer + concat_mla_absorb_q). Disabled under
|
||||
# async asserts: the fused path writes the pool directly and would
|
||||
@@ -411,6 +408,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _create_block_kv_indices(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
batch_size: int,
|
||||
max_blocks: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -430,11 +428,19 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
Returns:
|
||||
Block KV indices tensor
|
||||
"""
|
||||
dcp_enabled = get_parallel().dcp_enabled
|
||||
translated = self.kv_index_translator.is_translating and not dcp_enabled
|
||||
# A translated table holds the plan's page ids and reads the sink past
|
||||
# them: a draft extend pads every row's key length to its widest row,
|
||||
# and the padded queries (whose outputs are dropped) read that far.
|
||||
block_kv_indices = torch.full(
|
||||
(batch_size, max_blocks), -1, dtype=torch.int32, device=device
|
||||
(batch_size, max_blocks),
|
||||
0 if translated else -1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
if get_parallel().dcp_enabled:
|
||||
if dcp_enabled:
|
||||
self._fill_dcp_block_kv_indices(
|
||||
block_kv_indices,
|
||||
req_pool_indices,
|
||||
@@ -442,12 +448,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
)
|
||||
return block_kv_indices
|
||||
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
if translated:
|
||||
self.kv_index_translator.copy_page_table(plan, out=block_kv_indices)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
(
|
||||
@@ -480,13 +482,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
self.decode_cuda_graph_kv_indices = torch.full(
|
||||
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
# Unified pool: capture-stable buffer for the physical KV write loc, filled
|
||||
# out-of-graph in init_forward_metadata_out_graph so the in-graph
|
||||
# set_mla_kv_buffer captures no translate.
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.cuda_graph_out_cache_loc_kernel = torch.zeros(
|
||||
max_num_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
num_tokens_per_req = max_num_tokens // max_bs
|
||||
|
||||
if is_float4_e2m1fn_x2(self.data_type):
|
||||
@@ -603,6 +598,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -642,10 +638,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
# Update block indices for new sequences.
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.block_kv_indices,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
# The plan's table reaches a verify's draft tail.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=metadata.block_kv_indices[:bs]
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -755,6 +750,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
forward_batch.seq_lens.device,
|
||||
)
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
@@ -762,56 +758,13 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
)
|
||||
|
||||
# Out-of-graph on capture AND every replay-prep, so the in-graph
|
||||
# set_mla_kv_buffer captures no translate.
|
||||
if self.kv_index_translator.is_translating and (
|
||||
forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()
|
||||
):
|
||||
# The captured kernel consumes the whole buffer, so the tail a
|
||||
# shorter replay leaves must go to slot 0 rather than live pages.
|
||||
self._decode_kernel_loc = self.kv_index_translator.fill_capture_write_loc(
|
||||
out=self.cuda_graph_out_cache_loc_kernel,
|
||||
forward_batch=forward_batch,
|
||||
width=self.cuda_graph_out_cache_loc_kernel.numel(),
|
||||
)
|
||||
else:
|
||||
self._decode_kernel_loc = None
|
||||
|
||||
def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc:
|
||||
"""The loc an unfused KV scatter must write at: the capture-stable
|
||||
buffer under a captured unified-pool decode, since the translate
|
||||
rebinds `out_cache_loc` to a fresh tensor the graph never recorded;
|
||||
the batch's own loc everywhere else.
|
||||
"""
|
||||
if self._decode_kernel_loc is not None:
|
||||
# Filled by `fill_capture_write_loc`, which translates it.
|
||||
return KVWriteLoc(self._decode_kernel_loc, physical=True)
|
||||
return KVWriteLoc.for_batch(forward_batch)
|
||||
|
||||
def _resolve_fused_write_loc(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Write loc for the fused fp8 KV scatter, or None when this batch is
|
||||
not covered by it.
|
||||
|
||||
Captured decode refills `_decode_kernel_loc` out of the graph, and the
|
||||
captured kernel must read that buffer. Eager decode on a unified pool
|
||||
has no such buffer, and the caller falls back to the unfused path.
|
||||
"""
|
||||
if self._decode_kernel_loc is not None:
|
||||
return self._decode_kernel_loc
|
||||
return (
|
||||
None
|
||||
if self.kv_index_translator.is_translating
|
||||
else forward_batch.out_cache_loc
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Initialize the metadata for a forward pass."""
|
||||
if self._kv_shard_pool is not None:
|
||||
@@ -821,7 +774,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
forward_batch,
|
||||
)
|
||||
|
||||
self._decode_kernel_loc = None
|
||||
# Delegate to parent for non-decode modes.
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
@@ -912,6 +864,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
max_seqlen_pad = self._calc_padded_blocks(max_seq)
|
||||
block_kv_indices = self._create_block_kv_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
max_seqlen_pad,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -1287,7 +1240,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
# Fused: bf16->fp8 quantize + KV scatter + q concat
|
||||
# in one launch; None when not covered.
|
||||
@@ -1323,42 +1276,23 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert k is not None and k_rope is not None, (
|
||||
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
|
||||
)
|
||||
if self._decode_kernel_loc is not None:
|
||||
if merge_query and self._fused_set_kv_concat_q:
|
||||
# Fused: KV scatter + [q_nope | q_rope] concat in one
|
||||
# launch; None when the inputs are not covered.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=self._decode_kernel_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, self._kv_write_loc(forward_batch), k, k_rope
|
||||
)
|
||||
else:
|
||||
# eager (or static pool): out_cache_loc is physical.
|
||||
if (
|
||||
merge_query
|
||||
and self._fused_set_kv_concat_q
|
||||
and not self.kv_index_translator.is_translating
|
||||
):
|
||||
# Static pool only, conservatively.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=forward_batch.out_cache_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
# `out_cache_loc` is physical on every pool, eager or captured
|
||||
# (the runner's slot).
|
||||
if merge_query and self._fused_set_kv_concat_q:
|
||||
# Fused: KV scatter + [q_nope | q_rope] concat in one launch;
|
||||
# None when the inputs are not covered.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=forward_batch.out_cache_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
|
||||
# Prepare query tensor inline (already built when the fused save-KV
|
||||
# path ran)
|
||||
@@ -1515,7 +1449,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
# Fused: bf16->fp8 quantize + KV scatter + q concat
|
||||
# in one launch; None when not covered.
|
||||
@@ -1551,7 +1485,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
|
||||
)
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, self._kv_write_loc(forward_batch), k, k_rope
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
|
||||
# TODO refactor to avoid code duplication
|
||||
|
||||
@@ -604,6 +604,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
skip_attn_backend_init: Optional[bool] = None, # deprecated
|
||||
*,
|
||||
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
|
||||
return_kv_loc_plan: bool = False,
|
||||
) -> GenerationBatchResult:
|
||||
# Get forward batch from schedule batch
|
||||
if batch is not None:
|
||||
@@ -649,6 +650,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
expert_distribution_metrics=out.expert_distribution_metrics,
|
||||
routed_experts_output=out.routed_experts_output,
|
||||
indexer_topk_output=out.indexer_topk_output,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
|
||||
)
|
||||
|
||||
capture_pre_sample_logits(batch, forward_batch, logits_output)
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.utils.common import async_d2h as _async_d2h
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
|
||||
@@ -118,6 +119,11 @@ class GenerationBatchResult:
|
||||
# until the accepted path comes back over the relay.
|
||||
spec_verify_out_cache_loc: Optional[torch.Tensor] = None
|
||||
|
||||
# The plan the forward wrote its KV through, when the caller asked for it
|
||||
# (`return_kv_loc_plan`): a speculative worker's own writes into the same
|
||||
# slots take their ids from it.
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
|
||||
# PP+spec: [bs, spec_steps + 1] global node indices of the accepted path.
|
||||
# Every stage holds the KV for its own layers, so every stage has to compact
|
||||
# that path into its committed prefix; only the last stage can compute it.
|
||||
|
||||
@@ -510,10 +510,9 @@ class KVCacheConfigurator:
|
||||
token_to_kv_pool = None
|
||||
|
||||
# Unified-pool fast path: build req_to_token + token_to_kv pool + allocator
|
||||
# from one byte buffer, then return. Gated to the target worker by ROLE:
|
||||
# a compact-window DFLASH draft also passes req_to_token_pool=None, and
|
||||
# would allocate a SECOND unified byte buffer here. Supports hybrid
|
||||
# Mamba and hybrid SWA (not DSV4).
|
||||
# from one byte buffer, then return. Target worker only: a compact-window
|
||||
# DFLASH draft also passes req_to_token_pool=None. Supports hybrid Mamba
|
||||
# and hybrid SWA (not DSV4).
|
||||
if (
|
||||
get_memory().enable_unified_memory
|
||||
and req_to_token_pool is None
|
||||
@@ -581,7 +580,6 @@ class KVCacheConfigurator:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
placement = self._fused_draft_from_target_buffer(alloc)
|
||||
if placement is not None:
|
||||
# FUSED arm
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
draft_swa_layer_ids,
|
||||
)
|
||||
@@ -619,7 +617,6 @@ class KVCacheConfigurator:
|
||||
token_to_kv_pool_allocator=alloc,
|
||||
unified_memory_pool=None,
|
||||
)
|
||||
# PRIVATE arm
|
||||
draft_virtual_id_space = alloc.draft_virtual_id_space
|
||||
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
|
||||
"unified allocator virtual space smaller than the token "
|
||||
@@ -627,7 +624,7 @@ class KVCacheConfigurator:
|
||||
f"max_total_num_tokens={sizes.max_total_num_tokens}"
|
||||
)
|
||||
# Round UP to page alignment (paged draft backends view the
|
||||
# pool as (-1, page_size, H, D); the space is not aligned).
|
||||
# pool as (-1, page_size, H, D); the virtual space is not aligned).
|
||||
page = max(int(self.pool_page_size or 1), 1)
|
||||
draft_virtual_id_space = (
|
||||
(draft_virtual_id_space + page - 1) // page * page
|
||||
@@ -733,11 +730,8 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
def _fused_draft_from_target_buffer(self, alloc):
|
||||
"""The placement this draft binds to, or None for the private-pool
|
||||
fallback (the draft then owns a raw-virtual-indexed pool of its own).
|
||||
Target boot declines a placement whenever the fused region cannot
|
||||
serve the draft (`_fused_draft_decision`), so the private arm is the
|
||||
rollback lever, not a boot-order bug."""
|
||||
"""The placement this draft binds to, or None when the target placed
|
||||
none and the draft builds a private pool over the virtual id space."""
|
||||
if not (
|
||||
self.spec_algorithm.is_eagle() or self.spec_algorithm.is_dflash_family()
|
||||
):
|
||||
@@ -961,14 +955,10 @@ class KVCacheConfigurator:
|
||||
|
||||
def _fused_draft_decision(self):
|
||||
"""Whether, and where, the draft's layers fuse into the target's
|
||||
sub-pools. Fusion applies only for: unified memory ON, a unified target
|
||||
(hybrid-SWA or mamba hybrid, either full-pool kind), an EAGLE-family
|
||||
or DFLASH/DSPARK algorithm whose draft config was loaded at target
|
||||
boot. It declines, keeping the draft's private pool, for a non-MHA
|
||||
draft, a draft backend off the translated MHA rails, DCP, HiCache,
|
||||
host-pool decode retraction, or an explicit draft KV dtype unlike the
|
||||
host's; `place_fused_draft` then admits or declines the draft's layer
|
||||
kinds."""
|
||||
sub-pools. An empty decision means fusion does not apply (unified
|
||||
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
|
||||
loaded at target boot); a declined one says why the draft cannot fuse.
|
||||
The pool factories decide whether a decline refuses the boot."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
draft_kv_profile,
|
||||
@@ -998,10 +988,9 @@ class KVCacheConfigurator:
|
||||
"the fused region holds dense MHA K/V rows"
|
||||
)
|
||||
)
|
||||
# The backend(s) the draft runner will run. A DFLASH-family draft
|
||||
# worker resolves one as `build_draft_tp_worker` does; an EAGLE draft
|
||||
# runs the published draft backend, else the target's prefill/decode
|
||||
# pair. A published draft backend carries model-hook declarations.
|
||||
# The draft runner's backend, resolved as its worker will: an EAGLE
|
||||
# draft with no backend of its own runs the target's prefill/decode
|
||||
# pair. The published draft backend carries model-hook declarations.
|
||||
if self.spec_algorithm.is_dflash_family():
|
||||
from sglang.srt.speculative.draft_worker_common import (
|
||||
resolve_draft_worker_attention_backend,
|
||||
@@ -1066,10 +1055,9 @@ class KVCacheConfigurator:
|
||||
else 1
|
||||
)
|
||||
# A fused draft stores its rows in the host's KV dtype. An explicit
|
||||
# draft dtype resolves without the draft model, as the draft runner
|
||||
# resolves it, so a mismatch is declined here; `auto` follows the
|
||||
# draft's quant config, which only the draft runner knows, and is
|
||||
# checked when it binds.
|
||||
# draft dtype resolves here as the draft runner will; `auto` follows
|
||||
# the draft's quant config, which only the draft runner knows, so it
|
||||
# is checked when the draft binds.
|
||||
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
|
||||
if draft_kv_cache_dtype not in (None, "auto"):
|
||||
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
|
||||
@@ -1098,11 +1086,10 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
|
||||
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
|
||||
pad) for the boot solve's cell model. Single source of truth: assembled
|
||||
through the same spec the pool factory builds, so the priced entry and
|
||||
the allocated entry cannot drift. None when the draft does not fuse
|
||||
into that sub-pool."""
|
||||
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
|
||||
pad), built from the same spec as the pool factory's so the priced and
|
||||
allocated entries agree; None when the draft does not fuse into that
|
||||
sub-pool."""
|
||||
placement = self._fused_draft_decision().placement
|
||||
if placement is None or sub_pool_name != "full":
|
||||
return None
|
||||
@@ -1150,14 +1137,11 @@ class KVCacheConfigurator:
|
||||
|
||||
def _fused_draft_for_mamba_factory(self):
|
||||
"""`_fused_draft_for_pool_factory` for a mamba-hybrid host. Its unified
|
||||
buffer takes the WHOLE profiled KV budget (`unified_total_bytes`), so a
|
||||
private draft pool -- sized to the full virtual id span -- would be
|
||||
allocated on top of it, unbudgeted. Until the mamba solve prices that
|
||||
pool, an EAGLE-family or DFLASH draft that does not fuse is refused here
|
||||
instead of overcommitting GPU memory. A DSPARK draft that does not fuse
|
||||
keeps its private pool, unpriced: refusing it would refuse DSPARK on
|
||||
these hosts whenever its draft declines fusion, as the Kimi-Linear
|
||||
default (a trtllm_mha draft) does."""
|
||||
buffer takes the whole profiled KV budget, so a private draft pool
|
||||
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
|
||||
does not fuse is refused. A DSPARK draft that does not fuse keeps its
|
||||
private pool, unpriced, so DSPARK still boots where its draft declines
|
||||
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
|
||||
decision = self._fused_draft_decision()
|
||||
placement = self._fused_draft_for_pool_factory(decision)
|
||||
if (
|
||||
@@ -1169,18 +1153,16 @@ class KVCacheConfigurator:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory + EAGLE/EAGLE3/DFLASH on a mamba-hybrid "
|
||||
"target needs the draft's KV fused into the target's pages, but "
|
||||
f"this draft would keep a private pool ({reason}). A mamba host's "
|
||||
"unified buffer takes the whole KV budget, so a private draft "
|
||||
"pool would overcommit GPU memory."
|
||||
f"this draft cannot fuse ({reason}). A private draft pool would "
|
||||
"overcommit the KV budget the unified buffer already takes."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _fused_draft_for_swa_factory(self):
|
||||
"""`_fused_draft_for_pool_factory` for a hybrid-SWA host. Its boot solve
|
||||
prices a private EAGLE draft pool at the target's per-token size, not
|
||||
at the draft's own heads, head_dim and KV dtype, so a draft that does
|
||||
not fuse is refused here, as on a mamba host, instead of overcommitting
|
||||
GPU memory."""
|
||||
the draft's own geometry and KV dtype, so an EAGLE draft that does not
|
||||
fuse is refused rather than overcommit GPU memory."""
|
||||
decision = self._fused_draft_decision()
|
||||
placement = self._fused_draft_for_pool_factory(decision)
|
||||
if (
|
||||
@@ -1192,15 +1174,14 @@ class KVCacheConfigurator:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 on a hybrid-SWA target "
|
||||
"needs the draft's KV fused into the target's pages, but this "
|
||||
f"draft would keep a private pool ({reason}), which the unified "
|
||||
"pool's sizing does not price."
|
||||
f"draft cannot fuse ({reason}), and the unified pool's sizing "
|
||||
"does not price a private draft pool."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _fused_draft_for_pool_factory(self, decision):
|
||||
"""The placement a pool factory is handed, from ONE resolved decision,
|
||||
so the boot log reports exactly that placement: a declined fusion and
|
||||
an engaged one otherwise look identical from outside."""
|
||||
"""The placement a pool factory is handed. It logs the decision it
|
||||
returns, so the boot log shows whether fusion engaged or why not."""
|
||||
if decision.placement is None:
|
||||
if decision.declined is not None:
|
||||
logger.warning("fused draft KV disabled: %s", decision.declined)
|
||||
|
||||
@@ -23,18 +23,29 @@ A KV slot can be named in two id spaces:
|
||||
|
||||
The two coincide on a plain pool, so nothing here does any work there.
|
||||
|
||||
Backends get a `KVIndexTable`, which answers "what do I gather from, and
|
||||
which row is mine?":
|
||||
A pool can hold several sub-pools (full attention, sliding window), each with
|
||||
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
|
||||
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
|
||||
once, by the readers below and nowhere else; a backend never translates. The
|
||||
plan's read table exists only once a table reader asks for it (`read_table`,
|
||||
`copy_page_table`), built into that reader's own capture-stable table when it
|
||||
is the first; later readers take it, earlier ones translate in their own
|
||||
gather. The readers answer "what do I gather from, and which row is mine?"
|
||||
with a `KVIndexTable`:
|
||||
|
||||
ids[row_ids[b], pos]
|
||||
|
||||
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
|
||||
objects - no copy, no kernel)
|
||||
unified : ids = a built array of physical ids,
|
||||
row_ids = arange(batch_size)
|
||||
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
|
||||
objects - no copy, no kernel)
|
||||
shared table : ids = the plan's array of physical page ids,
|
||||
row_ids = arange(batch_size)
|
||||
no table : ids = req_to_token, row_ids = req_pool_indices, and
|
||||
v2p = the sub-pool's page table, applied by the gather
|
||||
|
||||
Backends call their own copy a *page table* (fa3) or a *block table*
|
||||
(trtllm); here it is the **index table**.
|
||||
(trtllm); here it is the **index table**. `read_source` serves a gather
|
||||
kernel, `pack_read_stream` a CSR stream, and `read_table` / `copy_page_table`
|
||||
a page-table kernel.
|
||||
|
||||
Converting only ever rewrites the page number and keeps the in-page offset, so
|
||||
one page-granular table serves both kinds of consumer: a block-table backend
|
||||
@@ -42,16 +53,15 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
|
||||
|
||||
token_id = entry * entry_page_size + pos % entry_page_size
|
||||
|
||||
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
|
||||
ids at ForwardBatch construction - the earliest consumer can snapshot it
|
||||
right after. The sliding-window write loc is derived at the same moment as
|
||||
read table, into the same index table.
|
||||
The one translation outside a plan is DCP's read production
|
||||
(`translate_dcp_read_ids`): which ids a rank reads is only known where the
|
||||
DCP index kernels select them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from typing import Optional, Tuple
|
||||
import functools
|
||||
from typing import Dict, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -63,6 +73,7 @@ from sglang.kernels.ops.kvcache.kv_read_table import (
|
||||
build_kv_read_table,
|
||||
build_kv_read_table_packed,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
@@ -71,30 +82,36 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
|
||||
)
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import (
|
||||
IdSpace,
|
||||
IdSpaceKind,
|
||||
KVLocPlan,
|
||||
pad_with_sink,
|
||||
window_read_extent,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
|
||||
class KVReadTables(msgspec.Struct, frozen=True):
|
||||
"""One capture-stable destination, however many id spaces the pool has.
|
||||
|
||||
A backend holds this and hands it back to `build_index_table(into=...)`;
|
||||
it never has to know whether there is a sliding-window space behind it.
|
||||
"""
|
||||
|
||||
full: torch.Tensor
|
||||
sliding_window: Optional[torch.Tensor]
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
class KVIndexTable(msgspec.Struct, frozen=True):
|
||||
"""Collection of what one batch gathers from."""
|
||||
"""What one batch gathers from, and how its entries become the ids a
|
||||
kernel reads: lane ``b``'s entry ``j`` is ``ids[row_ids[b] * row_stride +
|
||||
j // entry_page_size]``, expanded to tokens when an entry is a page, then
|
||||
mapped through ``v2p`` (page-granular, at the translator's page size) when
|
||||
one is given. A gather kernel executes exactly this; it never decides
|
||||
whether or how to translate."""
|
||||
|
||||
ids: torch.Tensor # 2-D array of KV ids to gather from
|
||||
row_ids: torch.Tensor # which row belongs to batch lane b
|
||||
row_stride: int # stride between rows of `ids`, in elements
|
||||
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
|
||||
is_translated: bool # entries are already physical ids
|
||||
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
|
||||
is_translated: bool # the ids a gather yields are physical
|
||||
# Virtual rows a gather translates on the way out (no shared table this
|
||||
# iteration); None when the entries are already what the kernel reads.
|
||||
v2p: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class KVIndexTranslator:
|
||||
@@ -108,6 +125,7 @@ class KVIndexTranslator:
|
||||
token_to_kv_pool,
|
||||
page_size: int,
|
||||
device: str,
|
||||
is_draft_worker: bool = False,
|
||||
):
|
||||
self.req_to_token = req_to_token
|
||||
self.page_size = page_size
|
||||
@@ -125,55 +143,104 @@ class KVIndexTranslator:
|
||||
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
|
||||
)
|
||||
self.is_translating = is_unified_target or is_fused_draft
|
||||
self._spaces: Dict[IdSpaceKind, IdSpace] = {}
|
||||
if self.is_translating:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
self._capture_page_size = alloc.page_size
|
||||
self._full_v2p_table = alloc.full_v2p_page_table
|
||||
self._full_p2v_table = alloc.full_p2v_page_table
|
||||
self._translate_full = alloc.translate_kv_loc
|
||||
# The WRITE loc is the one id that arrives DCP-WIDENED: read indices
|
||||
# are collapsed by the DCP index kernels, `out_cache_loc` still
|
||||
# carries the owner rule in `loc % dcp_size`. Identity with the read
|
||||
# translate when dcp_size == 1.
|
||||
self._translate_write_full = alloc.translate_write_loc
|
||||
# DCP read ids stay WIDENED to the consumer: selecting this rank's
|
||||
# share changes the length, so only the production site can do it.
|
||||
self.defer_read_translate = get_parallel().attn_dcp_size > 1
|
||||
# Keyed by the allocator, which owns the tables: a fused draft's
|
||||
# runner shares the target's full sub-pool, and so its key.
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(
|
||||
key=(IdSpaceKind.FULL, id(alloc)),
|
||||
# The WRITE loc is the one id that arrives DCP-WIDENED: read
|
||||
# indices are collapsed by the DCP index kernels, `out_cache_loc`
|
||||
# still carries the owner rule in `loc % dcp_size`.
|
||||
write=alloc.translate_write_loc,
|
||||
read_v2p=None if self.defer_read_translate else self._full_v2p_table,
|
||||
)
|
||||
routes_window_layers = is_unified_target or isinstance(
|
||||
token_to_kv_pool, BaseSWAKVPool
|
||||
)
|
||||
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
|
||||
self._swa_v2p_table = alloc.swa_v2p_page_table
|
||||
self._swa_write_loc_from_full = self._swa_write_loc_unified
|
||||
else:
|
||||
self._swa_v2p_table = None
|
||||
self._swa_write_loc_from_full = None
|
||||
# Straight from the virtual ids through the sliding-window
|
||||
# sub-pool's own table.
|
||||
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
|
||||
key=(IdSpaceKind.SLIDING_WINDOW, id(alloc)),
|
||||
write=alloc.translate_loc_from_full_to_swa,
|
||||
read_v2p=(
|
||||
None if self.defer_read_translate else alloc.swa_v2p_page_table
|
||||
),
|
||||
)
|
||||
else:
|
||||
self._capture_page_size = page_size
|
||||
self._full_v2p_table = None
|
||||
self._full_p2v_table = None
|
||||
self._translate_full = None
|
||||
self._translate_write_full = None
|
||||
self.defer_read_translate = False
|
||||
self._swa_v2p_table = None
|
||||
parallel = get_parallel()
|
||||
if _is_npu and parallel.dcp_enabled and not is_draft_worker:
|
||||
# An NPU DCP target writes its rank's share of the window in
|
||||
# rank-local slots (-1: another rank's); the scheduler,
|
||||
# `req_to_token` and the replicated draft pool keep the
|
||||
# allocator-global ones the window is planned in.
|
||||
localize = functools.partial(
|
||||
localize_dcp_indices,
|
||||
dcp_size=parallel.dcp_size,
|
||||
dcp_rank=parallel.dcp_rank,
|
||||
interleave_size=page_size,
|
||||
)
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(
|
||||
key=(IdSpaceKind.FULL, "dcp-rank-local"), write=localize
|
||||
)
|
||||
else:
|
||||
localize = None
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(key=(IdSpaceKind.FULL, None))
|
||||
# `translate_loc_from_full_to_swa` is abstract on `BaseSWAKVPool`,
|
||||
# which is also what the backends' `_resolve_swa_kv_pool` keys on.
|
||||
self._swa_write_loc_from_full = (
|
||||
token_to_kv_pool.translate_loc_from_full_to_swa
|
||||
if isinstance(token_to_kv_pool, BaseSWAKVPool)
|
||||
and (
|
||||
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
or token_to_kv_pool.request_window is None
|
||||
# Reads stay in `req_to_token`; the backends map them themselves.
|
||||
if isinstance(token_to_kv_pool, BaseSWAKVPool) and (
|
||||
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
or token_to_kv_pool.request_window is None
|
||||
):
|
||||
to_swa = token_to_kv_pool.translate_loc_from_full_to_swa
|
||||
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
|
||||
key=(IdSpaceKind.SLIDING_WINDOW, id(token_to_kv_pool)),
|
||||
write=(
|
||||
to_swa
|
||||
if localize is None
|
||||
else lambda ids: to_swa(localize(ids))
|
||||
),
|
||||
)
|
||||
else None
|
||||
)
|
||||
|
||||
self._rows: Optional[torch.Tensor] = (
|
||||
torch.arange(req_to_token.shape[0], dtype=torch.int64, device=device)
|
||||
if self.is_translating
|
||||
else None
|
||||
)
|
||||
self._index_table_memo: Optional[Tuple[weakref.ref, KVIndexTable]] = None
|
||||
|
||||
def space(self, kind: IdSpaceKind) -> Optional[IdSpace]:
|
||||
"""How this runner's ``kind`` sub-pool names the iteration's slots, or
|
||||
None when its pool has no such sub-pool."""
|
||||
return self._spaces.get(kind)
|
||||
|
||||
def write_ids(
|
||||
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""The ids ``forward_batch`` writes in this runner's ``kind`` sub-pool,
|
||||
taken from its plan and padded with the sink to its `out_cache_loc`'s
|
||||
length (a padded batch, a captured graph's buffer). None when there is
|
||||
no write loc or no such sub-pool."""
|
||||
plan = getattr(forward_batch, "kv_loc_plan", None)
|
||||
out_cache_loc = getattr(forward_batch, "out_cache_loc", None)
|
||||
if plan is None or out_cache_loc is None:
|
||||
return None
|
||||
ids = plan.write_ids(
|
||||
self, kind=kind, cols=getattr(forward_batch, "kv_loc_cols", None)
|
||||
)
|
||||
return pad_with_sink(ids, out_cache_loc.shape[0])
|
||||
|
||||
def capture_token_capacity(self, max_token_pool_size: int) -> int:
|
||||
"""Host capture rows are indexed by request-token IDs, not kernel IDs.
|
||||
@@ -185,240 +252,305 @@ class KVIndexTranslator:
|
||||
return self._full_v2p_table.numel() * self._capture_page_size
|
||||
return max_token_pool_size + self.page_size
|
||||
|
||||
def full_flat_v2p(self) -> Optional[torch.Tensor]:
|
||||
"""The full-side v2p page table for a kernel that translates flat ids
|
||||
itself, or ``None`` when this runner does not translate (pass-through
|
||||
and static pools)."""
|
||||
if not self.is_translating:
|
||||
return None
|
||||
return self._full_v2p_table
|
||||
# -- the iteration's plan --------------------------------------------------
|
||||
|
||||
# -- per-batch view --------------------------------------------------------
|
||||
def plan(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
) -> KVLocPlan:
|
||||
"""This iteration's ids, translated once. ``write_virtual`` is the
|
||||
iteration's write window (``[batch, window]`` flattened when its
|
||||
forwards write columns of it); ``read_extent`` is how far past
|
||||
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
|
||||
return KVLocPlan(
|
||||
source=self,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=write_virtual,
|
||||
read_extent=read_extent,
|
||||
write_slots=write_slots,
|
||||
)
|
||||
|
||||
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
|
||||
"""The plan of a forward that is its own iteration, from its own
|
||||
fields: its ``out_cache_loc`` is the write window (virtual, or with
|
||||
``runner_slots`` a runner's own buffer), and its reads reach
|
||||
`window_read_extent` past its lengths."""
|
||||
write_ids = forward_batch.out_cache_loc
|
||||
seq_lens = getattr(forward_batch, "seq_lens", None)
|
||||
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
|
||||
encoder_lens = getattr(forward_batch, "encoder_lens", None)
|
||||
if (
|
||||
encoder_lens is not None
|
||||
and seq_lens is not None
|
||||
and self.reads_are_translated
|
||||
):
|
||||
# An encoder-decoder row holds its encoder tokens ahead of the
|
||||
# decoder's, and its self-attention reads past `seq_lens`.
|
||||
seq_lens = seq_lens + encoder_lens
|
||||
encoder_lens_cpu = getattr(forward_batch, "encoder_lens_cpu", None)
|
||||
seq_lens_cpu = (
|
||||
None
|
||||
if seq_lens_cpu is None or encoder_lens_cpu is None
|
||||
else seq_lens_cpu + torch.tensor(encoder_lens_cpu).to(seq_lens_cpu)
|
||||
)
|
||||
return self.plan(
|
||||
req_pool_indices=getattr(forward_batch, "req_pool_indices", None),
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=None if runner_slots else write_ids,
|
||||
write_slots=write_ids if runner_slots else None,
|
||||
read_extent=window_read_extent(
|
||||
getattr(forward_batch, "forward_mode", None),
|
||||
getattr(forward_batch, "spec_info", None),
|
||||
write_ids,
|
||||
getattr(forward_batch, "batch_size", 0),
|
||||
),
|
||||
)
|
||||
|
||||
def bind_runner_slots(self, forward_batch) -> None:
|
||||
"""Bind a batch a runner builds over its own buffers, to capture a
|
||||
graph or to warm up with. Its write ids are the runner's buffer, used
|
||||
as it is; its reads plan over the runner's own rows and lengths."""
|
||||
self.own_plan(forward_batch, runner_slots=True).bind(forward_batch, self)
|
||||
|
||||
def build_iteration_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
space: IdSpace,
|
||||
*,
|
||||
rows: Optional[int],
|
||||
previous: Optional[KVIndexTable],
|
||||
into: Optional[torch.Tensor] = None,
|
||||
) -> KVIndexTable:
|
||||
"""Build ``plan``'s page table in ``space`` for `KVLocPlan.read_table`:
|
||||
the passthrough where reads stay virtual, else one build over ``[0,
|
||||
seq_lens + read_extent)``. Rows past the plan's batch read the sink; a
|
||||
request wider than ``previous`` copies its rows rather than translating
|
||||
again. ``into``, the asking reader's capture-stable table, is built in
|
||||
place when wide enough and then serves as the plan's table."""
|
||||
if space.read_v2p is None:
|
||||
return self._passthrough_table(plan.req_pool_indices)
|
||||
bs = int(plan.req_pool_indices.numel())
|
||||
rows = max(bs, rows or 0)
|
||||
if previous is not None:
|
||||
out = torch.zeros(
|
||||
(rows, previous.ids.shape[1]), dtype=torch.int32, device=self.device
|
||||
)
|
||||
out[: previous.ids.shape[0]].copy_(previous.ids)
|
||||
else:
|
||||
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
|
||||
slc = plan.seq_lens_cpu
|
||||
if slc is not None and slc.numel() > 0:
|
||||
max_seq = int(slc.max()) + plan.read_extent
|
||||
width = min(max(-(-max_seq // self.page_size), 1), row_pages)
|
||||
elif into is not None:
|
||||
# No host lengths: a row's speculative headroom outruns the
|
||||
# captured table, which is as wide as its graph reads.
|
||||
width = min(into.shape[1], row_pages)
|
||||
else:
|
||||
width = row_pages
|
||||
# `zero_tail` writes the sink past each live prefix, so neither a
|
||||
# reused `into` nor an `empty` table keeps stale columns.
|
||||
if into is not None and into.shape[0] >= rows and into.shape[1] >= width:
|
||||
out = into[:rows, :width]
|
||||
else:
|
||||
out = torch.empty((rows, width), dtype=torch.int32, device=self.device)
|
||||
if rows > bs:
|
||||
out[bs:].zero_()
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=plan.req_pool_indices,
|
||||
seq_lens=plan.seq_lens,
|
||||
v2p=space.read_v2p,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out,
|
||||
zero_tail=True,
|
||||
seq_len_delta=plan.read_extent,
|
||||
)
|
||||
return KVIndexTable(
|
||||
ids=out,
|
||||
row_ids=self._rows[:rows],
|
||||
row_stride=out.stride(0),
|
||||
entry_page_size=self.page_size,
|
||||
is_translated=True,
|
||||
)
|
||||
|
||||
# -- readers ---------------------------------------------------------------
|
||||
|
||||
def _reads_translated(self, kind: IdSpaceKind) -> bool:
|
||||
space = self._spaces.get(kind)
|
||||
return space is not None and space.read_v2p is not None
|
||||
|
||||
def read_source(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
bs: int,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
) -> KVIndexTable:
|
||||
"""The row source a gather kernel reads lane ``b``'s ``kind`` ids from:
|
||||
the plan's table once a table reader has had it built, else the virtual
|
||||
`req_to_token` rows with ``v2p`` for the kernel's gather to apply (no
|
||||
table is built for it). The plain `req_to_token` passthrough where
|
||||
reads stay virtual (a static pool, or DCP)."""
|
||||
if not self._reads_translated(kind):
|
||||
return self._passthrough_table(req_pool_indices)
|
||||
if plan.has_read_table(kind):
|
||||
return self._plan_table(plan, kind=kind, rows=bs)
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=True,
|
||||
v2p=self.space(kind).read_v2p,
|
||||
)
|
||||
|
||||
def read_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
rows: Optional[int] = None,
|
||||
) -> KVIndexTable:
|
||||
"""The plan's ``kind`` page table, exactly ``rows`` rows when given
|
||||
(lanes past the plan's batch read the sink); the ``req_to_token``
|
||||
passthrough where reads stay virtual (a static or private pool, or DCP,
|
||||
where the producing kernel selects this rank's share)."""
|
||||
if self._reads_translated(kind):
|
||||
return self._plan_table(plan, kind=kind, rows=rows)
|
||||
return self._passthrough_table(plan.req_pool_indices)
|
||||
|
||||
def _plan_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
kind: IdSpaceKind,
|
||||
rows: Optional[int] = None,
|
||||
into: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
)
|
||||
return plan.read_table(kind=kind, rows=rows, into=into)
|
||||
|
||||
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=False,
|
||||
)
|
||||
|
||||
def pack_read_stream(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
|
||||
over; return whether they are physical. Packed from the plan's table
|
||||
once a table reader has had it built, else gathered and translated
|
||||
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
|
||||
(a static pool, or DCP) for the caller to finish; a static SWA pool can
|
||||
fuse its full->swa table into the gather as ``token_mapping``."""
|
||||
bs = int(seq_lens.numel())
|
||||
if self._reads_translated(kind) and not plan.has_read_table(kind):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
)
|
||||
# `seq_lens` sizes the batch: a caller may hold a wider
|
||||
# req_pool_indices (a padded graph buffer), whose extra lanes have
|
||||
# no length to bound.
|
||||
assert req_pool_indices.numel() >= bs, (
|
||||
f"pack_read_stream: {bs} stream lanes, "
|
||||
f"{req_pool_indices.numel()} request rows"
|
||||
)
|
||||
build_kv_read_table_packed(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
v2p=self.space(kind).read_v2p,
|
||||
indptr=indptr,
|
||||
page_size=self.page_size,
|
||||
max_tokens=out.numel(),
|
||||
out=out,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
return True
|
||||
# The plan's table or a passthrough: the entries are the stream's ids.
|
||||
src = self.read_source(
|
||||
plan, req_pool_indices=req_pool_indices, bs=bs, kind=kind
|
||||
)
|
||||
assert src.v2p is None, "pack_read_stream: a table-less translating read"
|
||||
assert token_mapping is None or not src.is_translated
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
src.ids,
|
||||
src.row_ids[:bs],
|
||||
seq_lens,
|
||||
indptr,
|
||||
kv_start_idx,
|
||||
out,
|
||||
src.row_stride,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return src.is_translated or token_mapping is not None
|
||||
|
||||
def copy_page_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
) -> None:
|
||||
"""Fill ``out``, a captured graph's page table (padded lanes read the
|
||||
sink), with the plan's ``kind`` table. The plan's first table reader
|
||||
builds straight into ``out``, which then serves as the plan's table; a
|
||||
later one copies it. Columns past the plan's width keep stale values,
|
||||
which kernels never read past their own lengths."""
|
||||
assert self._reads_translated(kind), (
|
||||
"copy_page_table: reads stay virtual here (a non-unified pool, or "
|
||||
"DCP, where the caller selects this rank's share itself)"
|
||||
)
|
||||
rows = out.shape[0]
|
||||
first = not plan.has_read_table(kind)
|
||||
table = self._plan_table(
|
||||
plan, kind=kind, rows=rows, into=out if first else None
|
||||
)
|
||||
if table.ids.data_ptr() == out.data_ptr():
|
||||
return # built in place, or this very buffer is the plan's table
|
||||
width = min(out.shape[1], table.ids.shape[1])
|
||||
out[:, :width].copy_(table.ids[:rows, :width])
|
||||
|
||||
# -- dispositions ----------------------------------------------------------
|
||||
|
||||
@property
|
||||
def reads_are_translated(self) -> bool:
|
||||
"""Whether a read this translator fills comes out physical. False
|
||||
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
|
||||
``translate_dcp_read_ids`` to finish."""
|
||||
return self.is_translating and not self.defer_read_translate
|
||||
|
||||
def fill_packed_read_stream(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
total_tokens: int,
|
||||
out: torch.Tensor,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
|
||||
report whether they came out translated.
|
||||
|
||||
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
|
||||
Unified: one fused gather-and-translate, so no caller needs a
|
||||
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
|
||||
resident token, a length the pool bounds.
|
||||
|
||||
``sliding_window`` selects the swa sub-pool's own id space, built from
|
||||
VIRTUAL ids and never chained through full-physical. A ``False`` return
|
||||
means the ids are still VIRTUAL: the DCP path defers translation to
|
||||
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
|
||||
table as ``token_mapping`` to fuse that translation into the gather.
|
||||
"""
|
||||
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
|
||||
# (the padded graph buffer), and the extra lanes have no length to bound.
|
||||
bs = int(seq_lens.numel())
|
||||
assert req_pool_indices.numel() >= bs, (
|
||||
f"fill_packed_read_stream: {req_pool_indices.numel()} req rows for "
|
||||
f"{bs} lengths"
|
||||
)
|
||||
req_pool_indices = req_pool_indices[:bs]
|
||||
|
||||
if not self.reads_are_translated:
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
self.req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
indptr,
|
||||
kv_start_idx,
|
||||
out,
|
||||
self.req_to_token.stride(0),
|
||||
ENTRY_PAGE_SIZE=1,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return token_mapping is not None
|
||||
|
||||
if sliding_window:
|
||||
assert self._swa_v2p_table is not None, (
|
||||
"fill_packed_read_stream: sliding_window on a pool with no swa sub-pool"
|
||||
)
|
||||
build_kv_read_table_packed(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._swa_v2p_table if sliding_window else self._full_v2p_table,
|
||||
indptr=indptr,
|
||||
page_size=self.page_size,
|
||||
max_tokens=total_tokens,
|
||||
out=out,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
return True
|
||||
|
||||
def build_index_table(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
max_pages: Optional[int] = None,
|
||||
into: Optional[KVReadTables] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> KVIndexTable:
|
||||
"""The one per-batch entry point.
|
||||
|
||||
Non-unified: the raw ``(req_to_token, req_pool_indices)`` passthrough,
|
||||
no tensor ops and no copies. Unified: fills each row's live prefix and
|
||||
returns the table WHOLE, so a caller needing a stable pointer (a
|
||||
captured graph bakes it) passes its own tables in ``into``;
|
||||
``into=None`` allocates of width ``max_pages`` instead.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). An eager caller's
|
||||
``max_pages`` must already cover the delta.
|
||||
"""
|
||||
if not self.is_translating or self.defer_read_translate:
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=False,
|
||||
sliding_window_ids=None,
|
||||
)
|
||||
|
||||
bs = int(req_pool_indices.numel())
|
||||
if into is not None:
|
||||
out_full = into.full
|
||||
out_swa = into.sliding_window
|
||||
# A caller-owned table may be padded wider than req_to_token's span
|
||||
# (trtllm_mla / flashmla pad to a page-count bound); the columns
|
||||
# past it have no source to read, so stop there.
|
||||
width = min(
|
||||
out_full.shape[1] if max_pages is None else max_pages,
|
||||
-(-self.req_to_token.shape[1] // self.page_size),
|
||||
)
|
||||
else:
|
||||
assert max_pages is not None, (
|
||||
"KVIndexTranslator.build_index_table: allocating needs max_pages "
|
||||
"(from the batch's seq_lens_cpu max)"
|
||||
)
|
||||
width = max_pages
|
||||
out_full = torch.zeros((bs, width), dtype=torch.int32, device=self.device)
|
||||
out_swa = (
|
||||
torch.zeros((bs, width), dtype=torch.int32, device=self.device)
|
||||
if self._swa_v2p_table is not None
|
||||
else None
|
||||
)
|
||||
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._full_v2p_table,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_full,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
if out_swa is not None:
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._swa_v2p_table,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_swa,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
return KVIndexTable(
|
||||
ids=out_full,
|
||||
row_ids=self._rows[:bs],
|
||||
row_stride=out_full.stride(0),
|
||||
entry_page_size=self.page_size,
|
||||
is_translated=True,
|
||||
sliding_window_ids=out_swa,
|
||||
)
|
||||
|
||||
def fill_read_table(
|
||||
self,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
sliding_window_out: Optional[torch.Tensor] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> None:
|
||||
"""`build_index_table(into=...)` for a caller that owns a bare block
|
||||
table rather than a KVReadTables: the page-table consumers read that
|
||||
table directly, its rows already being the index table's rows.
|
||||
|
||||
`sliding_window_out` fills the swa twin in the same pass, for a hybrid
|
||||
model whose kernels take two block tables.
|
||||
"""
|
||||
# `reads_are_translated`, not `is_translating`: under DCP the builder
|
||||
# returns the passthrough view and writes nothing, so `is_translating`
|
||||
# would let a caller keep a stale table and never hear about it.
|
||||
assert self.reads_are_translated, (
|
||||
"KVIndexTranslator.fill_read_table cannot fill a page table when "
|
||||
"reads stay virtual (a non-unified pool, or DCP, where the caller "
|
||||
"must select this rank's share itself)"
|
||||
)
|
||||
assert sliding_window_out is None or self._swa_v2p_table is not None, (
|
||||
"KVIndexTranslator.fill_read_table: asked for a sliding-window "
|
||||
"table on a pool with no swa sub-pool"
|
||||
)
|
||||
self.build_index_table(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
into=KVReadTables(full=out, sliding_window=sliding_window_out),
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
|
||||
"""Eager per-batch view, memoized in one slot keyed by batch identity
|
||||
so multi-consumer metadata builds share a build. The next batch
|
||||
replaces the slot; consumers only read during their own build. The
|
||||
captured path does not memoize -- it refreshes its buffers per
|
||||
replay."""
|
||||
memo = self._index_table_memo
|
||||
if memo is not None and memo[0]() is forward_batch:
|
||||
return memo[1]
|
||||
max_pages = None
|
||||
if self.is_translating:
|
||||
# `seq_lens_cpu` is a non-None but STALE slice on a gpu_only
|
||||
# batch; `seq_lens_sum` is the signal that it is live. A stale max
|
||||
# under-sizes the table and the tail then reads as the sink.
|
||||
slc = forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is not None
|
||||
and slc is not None
|
||||
and slc.numel() > 0
|
||||
):
|
||||
max_seq = int(slc.max())
|
||||
else:
|
||||
max_seq = self.req_to_token.shape[1]
|
||||
max_pages = max(-(-max_seq // self.page_size), 1)
|
||||
view = self.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
max_pages=max_pages,
|
||||
)
|
||||
self._index_table_memo = (weakref.ref(forward_batch), view)
|
||||
return view
|
||||
return self._reads_translated(IdSpaceKind.FULL)
|
||||
|
||||
@property
|
||||
def full_v2p_table(self) -> Optional[torch.Tensor]:
|
||||
@@ -426,43 +558,11 @@ class KVIndexTranslator:
|
||||
pool needs no translation.
|
||||
|
||||
For the DCP page-table builders, whose gather is over a rank's cyclic
|
||||
slice rather than a row prefix, so `build_index_table` cannot serve
|
||||
them.
|
||||
slice rather than a row prefix, so a plan's row-prefix table cannot
|
||||
serve them.
|
||||
"""
|
||||
return self._full_v2p_table
|
||||
|
||||
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
|
||||
"""Whole-sequence-verify view: every row's live prefix widened by
|
||||
`seq_len_delta` columns (the drafts are read back from the pool).
|
||||
Not memoized -- verify is one consumer, and the batch's memoized
|
||||
prefix table stays valid for the others.
|
||||
|
||||
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
|
||||
carries `seq_lens_cpu` already expanded by the window and keeps the
|
||||
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
|
||||
lens would count the window twice. The width never exceeds the
|
||||
`req_to_token` row it is read from."""
|
||||
max_pages = None
|
||||
if self.is_translating:
|
||||
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
|
||||
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
|
||||
slc = live if live is not None else forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is not None
|
||||
and slc is not None
|
||||
and slc.numel() > 0
|
||||
):
|
||||
max_seq = int(slc.max()) + seq_len_delta
|
||||
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
|
||||
else:
|
||||
max_pages = row_pages
|
||||
return self.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
max_pages=max_pages,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def bind_and_verify_backends(self, backends) -> None:
|
||||
"""Boot: make every reachable backend carry THIS translator.
|
||||
|
||||
@@ -481,84 +581,13 @@ class KVIndexTranslator:
|
||||
"copy, not build its own."
|
||||
)
|
||||
|
||||
# -- write loc (phase 1; phase 2 lives in build_index_table) ----------------
|
||||
def bind_own_plan(self, forward_batch) -> None:
|
||||
"""Bind a batch built outside `ForwardBatch.init_new` that is its own
|
||||
iteration to a plan of its own fields (`own_plan`). A forward that
|
||||
shares its iteration's window takes that plan instead."""
|
||||
self.own_plan(forward_batch).bind(forward_batch, self)
|
||||
|
||||
def rebind_write_loc(self, forward_batch) -> None:
|
||||
"""Phase 1 of the WRITE contract: translate the batch's write loc to
|
||||
FULL-side physical ids exactly once, at ForwardBatch construction, and
|
||||
mark it physical. On non-unified pools the allocation is already
|
||||
physical, so only the mark is set.
|
||||
|
||||
REBIND, never mutate: the translate returns a FRESH tensor, so the
|
||||
ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept /
|
||||
in-flight machinery that reads it. The pre-translate tensor stays on
|
||||
the batch for `fill_capture_write_loc`.
|
||||
"""
|
||||
self._index_table_memo = None
|
||||
if forward_batch.out_cache_loc is None:
|
||||
return
|
||||
if self.is_translating:
|
||||
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
|
||||
forward_batch.out_cache_loc = self._translate_write_full(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
forward_batch.out_cache_loc_is_physical = True
|
||||
|
||||
def fill_capture_write_loc(
|
||||
self,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
forward_batch,
|
||||
width: Optional[int] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Translate this batch's WRITE loc straight into ``out``, a backend's
|
||||
capture-stable buffer, and return the live ``[:n]`` view. One launch
|
||||
fills the live prefix and clears the tail a shorter replay leaves;
|
||||
None when this pool needs no translation.
|
||||
|
||||
Must run at metadata-init time: `out` is reused every step, so filling
|
||||
it sooner would race a still-pending previous step under overlap
|
||||
scheduling.
|
||||
"""
|
||||
if not self.is_translating:
|
||||
return None
|
||||
virtual = forward_batch.out_cache_loc_virtual
|
||||
if virtual is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is None:
|
||||
return None
|
||||
# The runner builds the capture batch outside `init_new`, so no
|
||||
# rebind marked its virtual source; bake it holding sink ids.
|
||||
width = int(loc.numel()) if width is None else int(width)
|
||||
out[:width].zero_()
|
||||
return out[: int(loc.numel())]
|
||||
n = int(virtual.numel())
|
||||
width = n if width is None else int(width)
|
||||
buf = out[:width]
|
||||
self._translate_write_full(virtual, out=buf, out_width=width)
|
||||
return buf[:n]
|
||||
|
||||
def sliding_window_write_loc_for(
|
||||
self, out_cache_loc: Optional[torch.Tensor]
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""This batch's sliding-window write loc, or None when there is no loc
|
||||
this forward or the pool has no sliding-window id space."""
|
||||
if out_cache_loc is None or self._swa_write_loc_from_full is None:
|
||||
return None
|
||||
return self._swa_write_loc_from_full(out_cache_loc)
|
||||
|
||||
def _swa_write_loc_unified(self, full_loc: torch.Tensor) -> torch.Tensor:
|
||||
"""Sliding-window write loc, derived pointwise from FULL-side physical
|
||||
ids (phase 2 of the write contract).
|
||||
"""
|
||||
ps = self.page_size
|
||||
offset = full_loc % ps # == virtual_token % page_size
|
||||
# An unmapped physical page reads back as -1; clamp it rather than let
|
||||
# the gather wrap onto the v2p table's last element.
|
||||
virt_page = self._full_p2v_table[full_loc // ps].clamp_(min=0)
|
||||
return (self._swa_v2p_table[virt_page] * ps + offset).clamp_(min=0)
|
||||
|
||||
# -- token-level translate surface (the mixin / local-attn consumers) ------
|
||||
# -- DCP read production ---------------------------------------------------
|
||||
|
||||
@property
|
||||
def needs_read_translate(self) -> bool:
|
||||
@@ -575,14 +604,6 @@ class KVIndexTranslator:
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
if dcp_size > 1:
|
||||
widened_ids = widened_ids // dcp_size
|
||||
return self.translate_full_attn_ids(widened_ids)
|
||||
|
||||
def translate_full_attn_ids(
|
||||
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""Virtual token ids -> physical full-attention ids (the identity
|
||||
when no translation is needed, so callers never branch)."""
|
||||
if not self.is_translating:
|
||||
assert out is None, "passthrough translate takes no out="
|
||||
return kv_indices
|
||||
return self._translate_full(kv_indices, out=out)
|
||||
return widened_ids
|
||||
return self._translate_full(widened_ids)
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""One scheduler iteration's KV slot ids, per sub-pool.
|
||||
|
||||
Under the unified pool a KV slot has a virtual id (what the scheduler allocates
|
||||
and `req_to_token` stores) and, in each sub-pool, a physical id (what that
|
||||
sub-pool's kernels index). Every forward of one iteration -- the target's, and
|
||||
a fused draft's, which lives in the target's pages -- writes the same slots and
|
||||
reads the same rows, and compaction does not move a page while the iteration's
|
||||
forwards are in flight. So:
|
||||
|
||||
* The write window is translated once per sub-pool, by the plan: the
|
||||
full-attention ids when the plan is built, another sub-pool's on first
|
||||
use (`write_ids`). Every writer binds them.
|
||||
* A read is translated exactly once, only by the translator's read
|
||||
primitives (`KVIndexTranslator`); a consumer never translates.
|
||||
* The read table (`read_table`) exists only once a table reader asks for
|
||||
it, built into that reader's own capture-stable table when it is the
|
||||
first. Later readers take it; earlier ones translate in their own gather.
|
||||
* A sub-pool is named by its `IdSpace`; one that indexes virtual ids as they
|
||||
are (a static pool's, a private draft pool's) translates nothing.
|
||||
|
||||
One plan per iteration: `ForwardBatch.init_new` builds it for a lone forward, a
|
||||
speculative worker as soon as the iteration's slots are allocated (shared by
|
||||
its draft and target forwards and direct KV writers), and a graph runner for a
|
||||
batch over its own buffers (`write_slots`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Hashable, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_index_translator import (
|
||||
KVIndexTable,
|
||||
KVIndexTranslator,
|
||||
)
|
||||
|
||||
|
||||
class IdSpaceKind(enum.Enum):
|
||||
"""A sub-pool of a runner's KV pool, as the consumers that index it name
|
||||
it."""
|
||||
|
||||
# Full-attention layers: the pool's own token ids.
|
||||
FULL = enum.auto()
|
||||
# The sliding-window layers of a hybrid pool: the same token ids through
|
||||
# the sliding-window sub-pool's table.
|
||||
SLIDING_WINDOW = enum.auto()
|
||||
|
||||
|
||||
class IdSpace(msgspec.Struct, frozen=True):
|
||||
"""How one sub-pool names the iteration's slots.
|
||||
|
||||
``write`` maps the iteration's virtual token ids to this sub-pool's ids
|
||||
(None: they index it as they are). ``read_v2p`` is the page table its reads
|
||||
go through (None: reads stay in `req_to_token`, for the consumer to
|
||||
finish). Runners whose pools share a sub-pool share its ``key``, and with
|
||||
it a plan's ids for it."""
|
||||
|
||||
key: Hashable
|
||||
write: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
read_v2p: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def window_read_extent(forward_mode, spec_info, write_ids, batch_size: int) -> int:
|
||||
"""How far past ``seq_lens`` a forward reads: the window it writes, when
|
||||
its ``seq_lens`` do not count that window yet. A verify reads at most
|
||||
``draft_token_num`` past them (a ragged verify writes fewer for some rows);
|
||||
a draft decode reads back its own earlier steps; a draft extend's lengths
|
||||
take its window only after its batch is built. Zero otherwise."""
|
||||
if spec_info is None or write_ids is None or not batch_size:
|
||||
return 0
|
||||
if forward_mode.is_target_verify():
|
||||
return spec_info.draft_token_num
|
||||
if forward_mode.is_decode() or forward_mode.is_draft_extend_v2():
|
||||
return write_ids.numel() // batch_size
|
||||
return 0
|
||||
|
||||
|
||||
class TokenSpan(msgspec.Struct, frozen=True):
|
||||
"""A contiguous run ``[start, stop)`` of a write window's flat tokens (one
|
||||
half of a two-batch-overlap split), which a reader takes as a view; a
|
||||
``slice`` in `Cols` names columns of every row instead."""
|
||||
|
||||
start: int
|
||||
stop: int
|
||||
|
||||
|
||||
# Part of a `[batch, window]` write window: a slice of its columns, a
|
||||
# contiguous run of its flat tokens, or a flat index into it (-1: the sink).
|
||||
Cols = Union[slice, TokenSpan, torch.Tensor]
|
||||
|
||||
|
||||
def pad_with_sink(ids: Optional[torch.Tensor], n: int) -> Optional[torch.Tensor]:
|
||||
"""``ids`` padded to ``n`` entries with the sink (id 0 in every space), for
|
||||
a batch whose write ids cover more lanes than its plan's (a padded batch,
|
||||
a captured graph's buffer)."""
|
||||
if ids is None or ids.shape[0] >= n:
|
||||
return ids
|
||||
padded = ids.new_zeros(n)
|
||||
padded[: ids.shape[0]].copy_(ids)
|
||||
return padded
|
||||
|
||||
|
||||
class KVLocPlan:
|
||||
"""The ids of one iteration, in every sub-pool, each computed once."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source: KVIndexTranslator,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self._source = source
|
||||
self.req_pool_indices = req_pool_indices
|
||||
# Committed lengths when the plan is built; every read this iteration
|
||||
# stays within `seq_lens + read_extent` (the window it writes).
|
||||
self.seq_lens = seq_lens
|
||||
self.seq_lens_cpu = seq_lens_cpu
|
||||
self.read_extent = read_extent
|
||||
full = source.space(IdSpaceKind.FULL)
|
||||
if write_slots is not None:
|
||||
# A runner's own write buffer (graph capture, warmup): already in
|
||||
# its pool's ids, naming the sink until a replay fills it, and kept
|
||||
# by address by a captured graph.
|
||||
assert write_virtual is None
|
||||
self.write_virtual = None if full.write is not None else write_slots
|
||||
self.write_physical = write_slots
|
||||
else:
|
||||
# Aliases the ScheduleBatch's tensor, which stays virtual for the
|
||||
# radix tree, the accept path and lazy compaction's in-flight write
|
||||
# set.
|
||||
self.write_virtual = write_virtual
|
||||
# FIXME: a speculative iteration builds its plan before its draft
|
||||
# runs, so this launch sits on the host path ahead of the draft's
|
||||
# graph, idle GPU time at low concurrency. Translate the window
|
||||
# inside the iteration's first captured forward instead, into a
|
||||
# capture-stable buffer the plan owns, which every later forward
|
||||
# and direct writer reads -- once per sub-pool, the sliding-window
|
||||
# ids (derived in `write_ids`) included.
|
||||
self.write_physical = (
|
||||
full.write(write_virtual)
|
||||
if full.write is not None and write_virtual is not None
|
||||
else write_virtual
|
||||
)
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
|
||||
full.key: self.write_physical
|
||||
}
|
||||
self._read_tables: Dict[Hashable, KVIndexTable] = {}
|
||||
|
||||
# -- writes ----------------------------------------------------------------
|
||||
|
||||
def bind(self, batch, reader: KVIndexTranslator, *, cols: Optional[Cols] = None):
|
||||
"""Give ``batch`` (a ForwardBatch or a stand-in) this plan, the part of
|
||||
the window it writes (``cols``), and its full-attention write ids in
|
||||
`reader`'s pool. The one way a forward gets its write ids; another
|
||||
sub-pool's come from `KVIndexTranslator.write_ids`."""
|
||||
batch.kv_loc_plan = self
|
||||
batch.kv_loc_cols = cols
|
||||
batch.out_cache_loc = self.write_ids(reader, cols=cols)
|
||||
batch.out_cache_loc_virtual = (
|
||||
self.virtual_write_ids(cols=cols)
|
||||
if self.is_translated_for(reader)
|
||||
else None
|
||||
)
|
||||
batch.out_cache_loc_is_physical = batch.out_cache_loc is not None
|
||||
|
||||
def bind_replay(self, batch, reader: KVIndexTranslator, *, slots: torch.Tensor):
|
||||
"""`bind`, for the batch a captured graph replays this iteration's
|
||||
forward with. Its write ids are ``slots``, the runner's capture-stable
|
||||
buffer that already holds this plan's write ids and names the sink past
|
||||
them. The virtual mirror stays with the live batch."""
|
||||
self.bind(batch, reader)
|
||||
batch.out_cache_loc = slots
|
||||
batch.out_cache_loc_virtual = None
|
||||
|
||||
def write_ids(
|
||||
self,
|
||||
reader: KVIndexTranslator,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
cols: Optional[Cols] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""The write window as `reader`'s ``kind`` sub-pool indexes it, or None
|
||||
when its pool has no such sub-pool. ``cols`` selects part of a
|
||||
``[batch, window]`` window, for a forward that writes only part of it:
|
||||
a slice of columns (a draft step ahead of the verify window it opens),
|
||||
or a flat index into the window, -1 naming the sink (a ragged verify's
|
||||
packed rows)."""
|
||||
space = reader.space(kind)
|
||||
if space is None:
|
||||
return None
|
||||
if space.key not in self._write_ids:
|
||||
self._write_ids[space.key] = self._window_in(space)
|
||||
return self._cols(self._write_ids[space.key], cols)
|
||||
|
||||
def _window_in(self, space: IdSpace) -> Optional[torch.Tensor]:
|
||||
if self.write_virtual is None:
|
||||
# No window this iteration, or a runner's own slots, which name
|
||||
# the sink in every other sub-pool too.
|
||||
if self.write_physical is None:
|
||||
return None
|
||||
return torch.zeros_like(self.write_physical)
|
||||
if space.write is None:
|
||||
return self.write_virtual
|
||||
return space.write(self.write_virtual)
|
||||
|
||||
def virtual_write_ids(self, *, cols: Optional[Cols] = None):
|
||||
"""The same columns in the virtual space: the ids the scheduler's
|
||||
bookkeeping (lazy compaction's in-flight write set, the radix tree)
|
||||
names."""
|
||||
return self._cols(self.write_virtual, cols)
|
||||
|
||||
def _cols(self, ids: Optional[torch.Tensor], cols: Optional[Cols]):
|
||||
if ids is None or cols is None:
|
||||
return ids
|
||||
if isinstance(cols, TokenSpan):
|
||||
return ids[cols.start : cols.stop]
|
||||
if isinstance(cols, torch.Tensor):
|
||||
return torch.where(cols >= 0, ids[cols.clamp(min=0)], 0)
|
||||
bs = int(self.req_pool_indices.numel())
|
||||
if bs == 1:
|
||||
return ids[cols]
|
||||
return ids.view(bs, -1)[:, cols].reshape(-1)
|
||||
|
||||
def cols_slice(self, cols: Optional[Cols], tokens: slice) -> Cols:
|
||||
"""The ``tokens`` of a forward that writes ``cols``: the columns of
|
||||
one half of a batch split in two (two-batch overlap). A contiguous
|
||||
half stays a `TokenSpan`; only columns spread over the rows become a
|
||||
flat index into the window. Tokens past the window are left out; the
|
||||
consumer pads them with the sink."""
|
||||
if isinstance(cols, torch.Tensor):
|
||||
return cols[tokens]
|
||||
window = (
|
||||
self.write_virtual
|
||||
if self.write_virtual is not None
|
||||
else self.write_physical
|
||||
)
|
||||
if cols is None or isinstance(cols, TokenSpan):
|
||||
base = 0 if cols is None else cols.start
|
||||
end = window.numel() if cols is None else cols.stop
|
||||
start, stop, step = tokens.indices(end - base)
|
||||
if step == 1:
|
||||
return TokenSpan(base + start, base + max(start, stop))
|
||||
index = torch.arange(window.numel(), device=window.device)
|
||||
return self._cols(index, cols)[tokens]
|
||||
|
||||
def reads_from(
|
||||
self,
|
||||
source: KVIndexTranslator,
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
read_extent: int,
|
||||
) -> KVLocPlan:
|
||||
"""This plan's write ids, with reads planned over ``source``'s own
|
||||
`req_to_token` rows and lengths -- a draft that reads a compact table
|
||||
of its own while writing the iteration's window. The write ids are
|
||||
shared, not translated again."""
|
||||
plan = KVLocPlan(
|
||||
source=source,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=None,
|
||||
read_extent=read_extent,
|
||||
)
|
||||
plan.write_virtual = self.write_virtual
|
||||
plan.write_physical = self.write_physical
|
||||
plan._write_ids = self._write_ids
|
||||
return plan
|
||||
|
||||
# -- reads -----------------------------------------------------------------
|
||||
|
||||
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
|
||||
"""Whether a reader has had the ``kind`` table built yet."""
|
||||
return self._source.space(kind).key in self._read_tables
|
||||
|
||||
def read_table(
|
||||
self,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
rows: Optional[int] = None,
|
||||
into: Optional[torch.Tensor] = None,
|
||||
) -> KVIndexTable:
|
||||
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
|
||||
when a table reader first asks (in place in ``into``, that reader's
|
||||
capture-stable table, when given) and shared by every later reader.
|
||||
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
|
||||
sink, appended by copying, never by translating again. Where reads stay
|
||||
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
|
||||
space = self._source.space(kind)
|
||||
table = self._read_tables.get(space.key)
|
||||
if table is None or (
|
||||
table.is_translated and rows is not None and table.ids.shape[0] < rows
|
||||
):
|
||||
table = self._source.build_iteration_table(
|
||||
self, space, rows=rows, previous=table, into=into
|
||||
)
|
||||
self._read_tables[space.key] = table
|
||||
else:
|
||||
assert into is None, "a destination goes to the table's first build"
|
||||
if rows is not None and table.ids.shape[0] > rows:
|
||||
# Built wider, for a captured graph's padded lanes: this reader's
|
||||
# kernels size their batch by the table's rows.
|
||||
table = msgspec.structs.replace(
|
||||
table, ids=table.ids[:rows], row_ids=table.row_ids[:rows]
|
||||
)
|
||||
return table
|
||||
|
||||
def is_read_by(self, reader: KVIndexTranslator) -> bool:
|
||||
"""Whether `reader` reads this plan's tables: it indexes the plan's
|
||||
physical ids and gathers the same `req_to_token` rows. A reader with a
|
||||
table of its own (a draft's compact `req_to_token`) plans its own
|
||||
reads."""
|
||||
return (
|
||||
self.is_translated_for(reader)
|
||||
and reader.req_to_token is self._source.req_to_token
|
||||
)
|
||||
|
||||
def is_translated_for(self, reader: KVIndexTranslator) -> bool:
|
||||
"""Whether `reader`'s pool indexes this plan's physical ids: the
|
||||
target's, and a fused draft's, which lives in the target's full
|
||||
sub-pool. A pass-through reader (a static pool, a private draft pool)
|
||||
indexes the virtual ids."""
|
||||
full = reader.space(IdSpaceKind.FULL)
|
||||
return full.write is not None and full.key == self._full_key
|
||||
@@ -5,45 +5,23 @@ The nesting, outermost first:
|
||||
sub-pool a named pool inside the unified pool, with its own grow direction
|
||||
and frontier. HOST is the role it plays when its entries also
|
||||
carry draft bytes; only "full" hosts a draft here.
|
||||
slot one token entry of that sub-pool. The draft's KV becomes extra
|
||||
parts of every slot, after the host's own parts at
|
||||
`draft_offset_in_entry()`, so ONE slot id per token covers target
|
||||
and draft: one allocation, one free, one whole-page relocation
|
||||
carry both.
|
||||
slot one token entry of that sub-pool. The draft's KV is extra parts of
|
||||
every slot, after the host's own parts, so ONE slot id per token
|
||||
covers target and draft: one allocation, one free, one whole-page
|
||||
relocation carry both.
|
||||
region the draft geometry inside the host's entries. A region is not a
|
||||
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
|
||||
lane one position in the region's layer dimension -- one draft layer of
|
||||
one runner, present in every slot.
|
||||
|
||||
Runners are the draft's execution copies, and the two head shapes place them
|
||||
oppositely. A replicated head gives EVERY runner all the draft's layers, so
|
||||
each needs its own lanes or the runners clobber each other's KV (3 runners x
|
||||
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
|
||||
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
|
||||
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
|
||||
either direction. Each runner owns a contiguous run of lanes, and the runs
|
||||
tile the region in runner order.
|
||||
A lane is not a layer. Runners are the draft's execution copies: a
|
||||
replicated head gives EVERY runner all the draft's layers, on lanes of its own
|
||||
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
|
||||
while a per-depth head (one block per predicted position) gives each runner
|
||||
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
|
||||
contiguous run of lanes, and the runs tile the region in runner order.
|
||||
|
||||
Three phases, three moments in boot, and what carries each:
|
||||
|
||||
PLAN on the TARGET, before any memory exists
|
||||
KVCacheConfigurator._fused_draft_decision()
|
||||
draft_kv_profile(draft_model_config) what the checkpoint asks for
|
||||
place_fused_draft(profile, runners) admit, or decline with a reason
|
||||
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
|
||||
|
||||
CARVE as the byte buffer is cut
|
||||
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
|
||||
init_unified_*_pools(fused_draft=...)
|
||||
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
|
||||
UnifiedKVPool.build_dense_draft_views()
|
||||
|
||||
BIND on each DRAFT runner, built after the target
|
||||
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
|
||||
bind_fused_draft(...) -> UnifiedDraftKVPool
|
||||
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
|
||||
|
||||
A draft whose layers need another sub-pool kind declines to a private pool.
|
||||
A draft whose layers need another sub-pool kind declines fusion as a whole.
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
@@ -57,8 +35,8 @@ from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
|
||||
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
|
||||
sub-pool. The draft is a separate checkpoint, so its head geometry and
|
||||
layer count differ from the host's; its K and V rows are two more parts of
|
||||
the host entry, indexed by the host's physical token id."""
|
||||
layer count can differ from the host's; its K and V rows are two more parts
|
||||
of the host entry, indexed by the host's physical token id."""
|
||||
|
||||
lane_num: int
|
||||
head_num: int
|
||||
@@ -174,7 +152,6 @@ def draft_state_layer_num(draft_model_config) -> int:
|
||||
mambaish = mambaish_config(draft_model_config)
|
||||
if mambaish is None:
|
||||
return 0
|
||||
# Optional HF attribute: only conv-chain MTP configs declare it.
|
||||
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
|
||||
return 0
|
||||
return len(mambaish.mamba2_cache_params.layers)
|
||||
@@ -217,8 +194,8 @@ def draft_kv_profile(
|
||||
|
||||
|
||||
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
|
||||
private pool. Neither means fusion simply does not apply."""
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
|
||||
fuse. Neither means fusion does not apply."""
|
||||
|
||||
placement: Optional[FusedDraftPlacement] = None
|
||||
declined: Optional[str] = None
|
||||
@@ -253,8 +230,8 @@ def place_fused_draft(
|
||||
) -> FusedDraftDecision:
|
||||
"""Assign every draft layer of every runner to the host sub-pool whose
|
||||
lifetime covers what the layer reads: a full-attention layer rides in
|
||||
``"full"``. A layer kind no host arm serves declines the whole draft to
|
||||
its private pool."""
|
||||
``"full"``. A layer kind no host sub-pool serves declines the whole
|
||||
draft."""
|
||||
counts, reason = _runner_layer_counts(profile, num_runners)
|
||||
if counts is None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
|
||||
@@ -1871,26 +1871,21 @@ class KVWriteLoc:
|
||||
|
||||
All location info lives here (in the attention metadata), NOT in the pool:
|
||||
- ``loc``: the generic per-token write location (``out_cache_loc``).
|
||||
PHYSICAL on every pool: by allocation on non-unified pools,
|
||||
rebound at ForwardBatch construction (``rebind_write_loc``) on
|
||||
the unified pool.
|
||||
PHYSICAL on every pool: by allocation on non-unified pools, the
|
||||
iteration's plan's write ids on the unified pool (``KVLocPlan.bind``).
|
||||
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
|
||||
otherwise); under the unified pool the translator derives it from the
|
||||
same rebound loc (``sliding_window_write_loc_for``).
|
||||
- ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the
|
||||
construction-time rebind it is the SAME id space as ``loc``, so pools
|
||||
fall back to ``loc`` when it is ``None`` -- only triton's captured path
|
||||
still passes its capture-stable
|
||||
``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a
|
||||
same-space alias slated for collapse).
|
||||
otherwise): the same plan's write ids in the sliding-window sub-pool
|
||||
(``KVIndexTranslator.write_ids``).
|
||||
- ``full_loc``: OPTIONAL full-attention-sub-pool location, in the same id
|
||||
space as ``loc``; pools fall back to ``loc`` when it is ``None``.
|
||||
|
||||
``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved
|
||||
loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``);
|
||||
``loc`` is the generic fallback. Bundling them lets a backend issue one
|
||||
``set_kv_buffer`` call regardless of pool type.
|
||||
|
||||
``physical`` marks the locs as physical token ids: the batch's write loc
|
||||
after ``rebind_write_loc``, or ids a backend translated itself. A unified
|
||||
``physical`` marks the locs as physical token ids: the write ids a plan
|
||||
bound to the batch, or ids produced separately and marked so. A unified
|
||||
pool's write door refuses a loc not marked physical; other pools do not
|
||||
check.
|
||||
"""
|
||||
@@ -1910,9 +1905,9 @@ class KVWriteLoc:
|
||||
) -> KVWriteLoc:
|
||||
"""The batch's ``out_cache_loc`` as a write loc, carrying the batch's
|
||||
physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under
|
||||
the same mark, so it must be derived from that rebound loc (as
|
||||
``sliding_window_write_loc_for`` does); a loc produced separately
|
||||
states its own mark with ``KVWriteLoc(loc, physical=...)``."""
|
||||
the same mark, so it must come from the same plan (as
|
||||
``KVIndexTranslator.write_ids`` does); a loc produced separately states
|
||||
its own mark with ``KVWriteLoc(loc, physical=...)``."""
|
||||
return cls(
|
||||
forward_batch.out_cache_loc,
|
||||
swa_loc,
|
||||
@@ -2095,9 +2090,9 @@ class KVCache(abc.ABC):
|
||||
if self.requires_physical_write_loc and not write_loc_is_physical(loc_info):
|
||||
raise ValueError(
|
||||
f"{where}: write loc is not marked physical. Hand the pool "
|
||||
"KVWriteLoc.for_batch(forward_batch) after "
|
||||
"KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, "
|
||||
"physical=True) for ids translated separately."
|
||||
"KVWriteLoc.for_batch(forward_batch) for a batch bound to a "
|
||||
"plan (KVLocPlan.bind), or KVWriteLoc(loc, physical=True) for "
|
||||
"ids translated separately."
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
@@ -4734,7 +4729,7 @@ class MLATokenToKVPool(KVCache):
|
||||
|
||||
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
|
||||
# False: this pool takes a WIDENED loc. The unified pool resolves it in
|
||||
# `KVIndexTranslator.rebind_write_loc` and flips this.
|
||||
# the plan's write translation (`translate_write_loc`) and flips this.
|
||||
write_loc_is_dcp_resolved = False
|
||||
|
||||
@property
|
||||
|
||||
@@ -15,15 +15,11 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
|
||||
"""Dense draft KV over the draft parts of one host sub-pool's entries.
|
||||
|
||||
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
|
||||
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
|
||||
``layer_lanes`` maps each of this runner's layer ids to its region lane.
|
||||
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
|
||||
produced by the allocator's translate (the id-space choke point binds it,
|
||||
see KVIndexTranslator); the pool exposes `host_allocator` for that
|
||||
binding. Relocation needs no method here: compaction moves whole page
|
||||
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
|
||||
raises so a stray per-slot move fails loudly instead of corrupting the
|
||||
fused layout.
|
||||
every slot of the host sub-pool; ``layer_lanes`` maps each of this
|
||||
runner's layer ids to its region lane. Locs are the target's PHYSICAL
|
||||
token ids, translated through `host_allocator`. Compaction moves whole
|
||||
page envelopes on the HOST pool, draft bytes included, so `move_kv_cache`
|
||||
raises rather than let a per-slot move corrupt the fused layout.
|
||||
"""
|
||||
|
||||
requires_physical_write_loc = True
|
||||
|
||||
@@ -149,8 +149,7 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
|
||||
|
||||
The slot stride stays the one entry, so host and draft views are indexed
|
||||
by the same physical token id; only their offsets differ. `draft_region
|
||||
is None` keeps the layout byte-identical to the unfused one.
|
||||
by the same physical token id; only their offsets differ.
|
||||
"""
|
||||
|
||||
head_num: int
|
||||
@@ -930,8 +929,9 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
# Lifetime owned by UnifiedKVPool; do not delete the views.
|
||||
pass
|
||||
|
||||
# `rebind_write_loc` already collapsed the widened id and sent the rows this
|
||||
# rank does not own to the padding sink.
|
||||
# The plan's write translation (`translate_write_loc`) already collapsed
|
||||
# the widened id and sent the rows this rank does not own to the padding
|
||||
# sink.
|
||||
write_loc_is_dcp_resolved = True
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
|
||||
@@ -814,7 +814,6 @@ class CPUGraphRunner:
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -832,6 +831,7 @@ class CPUGraphRunner:
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
global_forward_mode=self.capture_forward_mode,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
# Wrap all forward calls with capture_with_skip_cross_attention so that
|
||||
# mllama (and any other encoder-decoder model) sees the correct compile-
|
||||
# time constant for skip_cross_attention during tracing.
|
||||
|
||||
@@ -83,6 +83,7 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
):
|
||||
self.prefix_chunk_kv_indices = []
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
translator = get_attn_backend().kv_index_translator
|
||||
for idx in range(self.num_prefix_chunks):
|
||||
chunk_starts = self.prefix_chunk_starts[idx]
|
||||
chunk_seq_lens = self.prefix_chunk_seq_lens[idx]
|
||||
@@ -93,6 +94,18 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
num_chunk_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
if translator.reads_are_translated:
|
||||
# The plan's table already holds the chunk's physical ids.
|
||||
translator.pack_read_stream(
|
||||
self.kv_loc_plan,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=chunk_seq_lens,
|
||||
indptr=chunk_cu_seq_lens,
|
||||
out=chunk_kv_indices,
|
||||
kv_start_idx=chunk_starts,
|
||||
)
|
||||
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
|
||||
continue
|
||||
create_chunked_prefix_cache_kv_indices[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
@@ -107,7 +120,6 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
self.prefix_chunk_starts_cpu[idx],
|
||||
self.prefix_chunk_seq_lens_cpu[idx],
|
||||
)
|
||||
translator = get_attn_backend().kv_index_translator
|
||||
chunk_kv_indices = translator.translate_dcp_read_ids(chunk_kv_indices)
|
||||
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
|
||||
|
||||
@@ -266,19 +278,28 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
device=self.req_pool_indices.device,
|
||||
)
|
||||
kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0)
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
create_flashinfer_kv_indices_triton[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
self.seq_lens,
|
||||
kv_indptr,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.shape[1],
|
||||
)
|
||||
# None on a backend that never bound a translator.
|
||||
# None on a backend that never bound a translator, which reads virtual
|
||||
# ids; otherwise one gather from the iteration's plan (DCP takes its
|
||||
# own branch upstream).
|
||||
src = get_attn_backend().kv_index_translator
|
||||
if src is not None:
|
||||
kv_indices = src.translate_full_attn_ids(kv_indices)
|
||||
if src is None:
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
create_flashinfer_kv_indices_triton[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
self.seq_lens,
|
||||
kv_indptr,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.shape[1],
|
||||
)
|
||||
else:
|
||||
src.pack_read_stream(
|
||||
self.kv_loc_plan,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=self.seq_lens,
|
||||
indptr=kv_indptr,
|
||||
out=kv_indices,
|
||||
)
|
||||
self.mha_one_shot_kv_indices = kv_indices
|
||||
return kv_indices
|
||||
|
||||
@@ -43,7 +43,6 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
compute_req_all_ids_info,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_slot_in,
|
||||
@@ -77,6 +76,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
|
||||
@@ -192,21 +192,6 @@ def _elastic_should_preserve_local_token_counts(
|
||||
return uneven_token_count
|
||||
|
||||
|
||||
def _localize_npu_dcp_out_cache_loc(
|
||||
out_cache_loc: torch.Tensor,
|
||||
*,
|
||||
interleave_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map allocator-global NPU DCP slots to this target rank."""
|
||||
parallel = get_parallel()
|
||||
return localize_dcp_indices(
|
||||
out_cache_loc,
|
||||
parallel.dcp_size,
|
||||
parallel.dcp_rank,
|
||||
interleave_size,
|
||||
)
|
||||
|
||||
|
||||
class ForwardMode(IntEnum):
|
||||
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
|
||||
# It is also called "prefill" in common terminology.
|
||||
@@ -532,14 +517,23 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# The original sequence length without being chunked. Qwen-1M related.
|
||||
orig_seq_lens: Optional[torch.Tensor] = None
|
||||
|
||||
# The write loc before `rebind_write_loc` replaced it with physical
|
||||
# ids; a backend re-derives from it into its capture-stable buffer.
|
||||
# This iteration's KV ids, translated once per sub-pool (`KVLocPlan`).
|
||||
# Every forward of the iteration holds the same plan; `out_cache_loc` is
|
||||
# its write window in the full-attention ids this runner's pool indexes,
|
||||
# and a backend that writes another sub-pool asks the plan for its ids
|
||||
# (`KVIndexTranslator.write_ids`).
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
# The part of the plan's window this forward writes (None: all of it).
|
||||
kv_loc_cols: Optional[Cols] = None
|
||||
# The same write ids in the virtual space when `out_cache_loc` holds
|
||||
# translated ones (None on a pool that indexes virtual ids), for the
|
||||
# scheduler's bookkeeping: lazy compaction's in-flight write set, TBO's
|
||||
# split, state capture.
|
||||
out_cache_loc_virtual: Optional[torch.Tensor] = None
|
||||
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
|
||||
# consumed by the Ascend backend for PA_ND block tables. None elsewhere.
|
||||
out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None
|
||||
# Whether `out_cache_loc` holds physical ids: set by
|
||||
# KVIndexTranslator.rebind_write_loc; capture-time batches declare it.
|
||||
# Whether `out_cache_loc` holds physical ids: set by `KVLocPlan.bind`.
|
||||
out_cache_loc_is_physical: bool = False
|
||||
# The indices to track mamba state with
|
||||
mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64
|
||||
@@ -937,9 +931,16 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
return_hidden_states_before_norm: bool,
|
||||
extend_position_info=None,
|
||||
spec_mrope_positions: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
write_cols: Optional[Cols] = None,
|
||||
):
|
||||
# init_new must not mutate the input ScheduleBatch; per-forward
|
||||
# overrides go through explicit keyword arguments.
|
||||
#
|
||||
# `kv_loc_plan`: the iteration's plan when this forward is one of
|
||||
# several over the same slots (a speculative iteration), `write_cols`
|
||||
# the columns of its window this forward writes. Without one, this
|
||||
# forward is its own iteration and builds the plan itself.
|
||||
|
||||
# capture_hidden_mode=None means no override: capture the server's
|
||||
# configured maximum so lower-mode requests can share one graph.
|
||||
@@ -1051,15 +1052,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
spec_info=batch.spec_info,
|
||||
)
|
||||
|
||||
# ScheduleBatch and req_to_token keep allocator-global slot identities.
|
||||
# Preserve that view before exposing rank-local NPU DCP write slots.
|
||||
# ScheduleBatch and req_to_token keep allocator-global slot identities,
|
||||
# and so does the plan's window. An NPU DCP target writes rank-local
|
||||
# slots, which its translator maps the window to at `bind` below; keep
|
||||
# the global view for the consumers that read it.
|
||||
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
|
||||
ret.origin_out_cache_loc = ret.out_cache_loc
|
||||
if ret.out_cache_loc is not None:
|
||||
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
|
||||
ret.out_cache_loc,
|
||||
interleave_size=model_runner.page_size,
|
||||
)
|
||||
ret._maybe_init_non_generation_fields(batch)
|
||||
|
||||
device = model_runner.device
|
||||
@@ -1073,7 +1071,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
device,
|
||||
)
|
||||
|
||||
model_runner.kv_index_translator.rebind_write_loc(ret)
|
||||
translator = model_runner.kv_index_translator
|
||||
if kv_loc_plan is None:
|
||||
kv_loc_plan = translator.own_plan(ret)
|
||||
kv_loc_plan.bind(ret, translator, cols=write_cols)
|
||||
|
||||
if envs.SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.get():
|
||||
hashed = _hash_rids_to_tensor(
|
||||
@@ -2143,6 +2144,8 @@ def build_inner_fb_view(
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
# A caller may hand in another view that does not carry this field.
|
||||
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
|
||||
kv_loc_plan=getattr(forward_batch, "kv_loc_plan", None),
|
||||
kv_loc_cols=getattr(forward_batch, "kv_loc_cols", None),
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
spec_info=forward_batch.spec_info,
|
||||
|
||||
@@ -898,6 +898,7 @@ class ModelRunner:
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
page_size=self.page_size or 1,
|
||||
device=self.device,
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
)
|
||||
|
||||
def max_shared_logits_buffer_rows(self) -> int:
|
||||
|
||||
@@ -662,7 +662,6 @@ class BaseRunner(ABC):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=batch_size,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -698,6 +697,7 @@ class BaseRunner(ABC):
|
||||
global_forward_mode=capture_forward_mode,
|
||||
lora_ids=lora_ids,
|
||||
)
|
||||
mr.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
if buffers.ngram_embedding_info is not None:
|
||||
forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice(
|
||||
|
||||
@@ -189,6 +189,8 @@ def build_replay_fb_view(
|
||||
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
max_seq_len_override=forward_batch.max_seq_len_override,
|
||||
@@ -1005,7 +1007,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -1043,6 +1044,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
rids_int=rids_int,
|
||||
bootstrap_room_ids_int=bootstrap_room_ids_int,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
# Trip the coordinator so the hisparse code path is captured into the
|
||||
# graph; backends read it from self.model_runner.hisparse_coordinator.
|
||||
@@ -1118,9 +1120,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self._post_process_after_profile(prof)
|
||||
self._profiler = None
|
||||
|
||||
# No pool-side pin to clear: the captured full-physical write loc rides the
|
||||
# backend's `ForwardMetadata.out_cache_loc_full_physical` (-> KVWriteLoc.full_loc).
|
||||
|
||||
def _capture_one_stream(self, stream_idx: Optional[int] = None) -> None:
|
||||
avail_mem = get_available_gpu_memory(
|
||||
self.model_runner.device,
|
||||
@@ -1271,9 +1270,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if (c := self.model_runner.canary_manager) is not None
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
# Full-physical write loc lives in the attention metadata (the backend's
|
||||
# `out_cache_loc_full_physical` -> KVWriteLoc.full_loc), so the runner
|
||||
# wires no buffer here. (SWA write loc rides the `swa_out_cache_loc` rail.)
|
||||
|
||||
with canary_ctx:
|
||||
shape_key = self._make_graph_key(
|
||||
|
||||
@@ -1496,7 +1496,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
with torch.device(self.device):
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=_slot("input_ids"),
|
||||
input_embeds=(
|
||||
@@ -1562,6 +1561,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
|
||||
max_seq_len_override=self.max_context_size,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
ngram_manager = self.model_runner.ngram_embedding_manager
|
||||
if ngram_manager.enabled:
|
||||
forward_batch.ngram_embedding_info = NgramEmbeddingInfo.create(
|
||||
@@ -1861,7 +1861,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
static_forward_batch = ForwardBatch(
|
||||
forward_mode=pcg_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
@@ -1926,6 +1925,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
),
|
||||
max_seq_len_override=self.max_context_size,
|
||||
)
|
||||
forward_batch.kv_loc_plan.bind_replay(
|
||||
static_forward_batch,
|
||||
self.model_runner.kv_index_translator,
|
||||
slots=out_cache_loc,
|
||||
)
|
||||
# The n-gram hasher runs outside the graph and reads this at replay.
|
||||
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
|
||||
static_forward_batch.engram_history = forward_batch.engram_history
|
||||
|
||||
@@ -803,8 +803,8 @@ class DSparkDraftMixin:
|
||||
attn.attn.v_scale,
|
||||
)
|
||||
else:
|
||||
# Translated by the injector (physical by allocation on a plain
|
||||
# pool).
|
||||
# `cache_loc` comes from `TargetHiddenKvInjector.ids_for`,
|
||||
# already in the ids the draft pool indexes.
|
||||
pool.set_kv_buffer(
|
||||
attn.attn,
|
||||
KVWriteLoc(cache_loc, physical=True),
|
||||
|
||||
@@ -481,12 +481,8 @@ class InklingAttention(nn.Module):
|
||||
do_store = do_bf16_store or do_mxfp8_store
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
# SWA sub-pool + the backend's full->SWA translated write location.
|
||||
# SWA sub-pool + the plan's sliding-window write ids (backend buffer).
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
es = q.element_size()
|
||||
@@ -584,10 +580,6 @@ class InklingAttention(nn.Module):
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
is_v2 = forward_batch.forward_mode.is_draft_extend_v2()
|
||||
@@ -691,10 +683,6 @@ class InklingAttention(nn.Module):
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
es = q.element_size()
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
@@ -51,6 +52,10 @@ class DFlashVerifyInput(SpecInput):
|
||||
live_seq_lens_cpu: Optional[torch.Tensor] = None
|
||||
# Conservative request-lifetime bound for candidate graph dispatch.
|
||||
candidate_max_seq_len_upper_bound: Optional[int] = None
|
||||
# The iteration's plan when the caller planned the verify window, and the
|
||||
# part of it this verify writes (`KVLocPlan.write_ids`).
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
kv_loc_cols: Optional[Cols] = None
|
||||
|
||||
def __post_init__(self):
|
||||
super().__init__(spec_input_type=SpecInputType.DFLASH_VERIFY)
|
||||
@@ -101,6 +106,8 @@ class DFlashVerifyInput(SpecInput):
|
||||
target_worker.model_runner,
|
||||
capture_hidden_mode=self.capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=self.kv_loc_plan,
|
||||
write_cols=self.kv_loc_cols,
|
||||
)
|
||||
|
||||
can_run_cuda_graph = bool(
|
||||
@@ -133,13 +140,14 @@ class DFlashVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
plan: KVLocPlan,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kv_indices_buf: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
device = req_pool_indices.device
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
@@ -179,14 +187,14 @@ class DFlashVerifyInput(SpecInput):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
mask = self.custom_mask
|
||||
if mask is not None:
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -1744,6 +1745,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
return out_tokens
|
||||
|
||||
def _draft_write_ids(self, plan: KVLocPlan) -> torch.Tensor:
|
||||
"""The plan's write ids as the draft pool indexes them: the target's
|
||||
physical ids for a fused draft, the virtual ids for a private one."""
|
||||
return plan.write_ids(self.draft_model_runner.kv_index_translator)
|
||||
|
||||
def _append_target_hidden_to_draft_kv_by_loc(
|
||||
self,
|
||||
*,
|
||||
@@ -1845,17 +1851,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
if commit_lens.dtype != torch.int32:
|
||||
commit_lens = commit_lens.to(torch.int32)
|
||||
|
||||
# On a translating pool the translate writes a new tensor, so the
|
||||
# callers' locs stay virtual: the post-verify 2-D buffer is re-read as
|
||||
# virtual ids by the compact req_to_token rebuild. On a plain pool it
|
||||
# returns `cache_loc` itself.
|
||||
translator = self.draft_model_runner.kv_index_translator
|
||||
cache_loc = translator.translate_full_attn_ids(cache_loc)
|
||||
if cache_loc_2d is not None:
|
||||
# Same ids, so reshape the translated copy rather than translate
|
||||
# them a second time.
|
||||
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
|
||||
|
||||
# `cache_loc` arrives in the draft pool's id space (`_draft_write_ids`);
|
||||
# the callers' window buffers stay virtual for the compact
|
||||
# req_to_token rebuild.
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
@@ -2265,7 +2263,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
logits_output, next_token_ids = (
|
||||
batch_output.logits_output,
|
||||
@@ -2316,9 +2316,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
ctx_lens,
|
||||
int(sum(batch.extend_lens)),
|
||||
)
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
self._append_target_hidden_to_draft_kv_by_loc(
|
||||
target_hidden=logits_output.hidden_states,
|
||||
cache_loc=batch.out_cache_loc,
|
||||
cache_loc=self._draft_write_ids(kv_loc_plan),
|
||||
positions=positions,
|
||||
extend_lens=ctx_lens,
|
||||
)
|
||||
@@ -2494,6 +2495,19 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
positions = positions_2d.reshape(-1)
|
||||
verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1)
|
||||
# The iteration's window -- the draft block, the verify and the
|
||||
# target-hidden KV writes all write it -- planned once.
|
||||
kv_loc_plan = self.target_worker.model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=(
|
||||
batch.seq_lens_cpu
|
||||
if batch.seq_lens_cpu is not None
|
||||
else draft_input.nxt_kv_lens_cpu
|
||||
),
|
||||
write_virtual=verify_out_cache_loc,
|
||||
read_extent=block_size,
|
||||
)
|
||||
|
||||
seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs]
|
||||
if self.use_compact_draft_cache:
|
||||
@@ -2555,10 +2569,20 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
),
|
||||
global_num_token_non_padded_cpu=bs * block_size,
|
||||
)
|
||||
# Hand-built draft batch bypasses ForwardBatch.init_new: under the
|
||||
# unified pool the write loc must be rebound to the draft's
|
||||
# kernel-facing ids here (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(forward_batch)
|
||||
# The draft block writes the whole window. A compact draft reads its
|
||||
# own `req_to_token` rows over its own lengths.
|
||||
draft_translator = self.draft_model_runner.kv_index_translator
|
||||
draft_plan = (
|
||||
kv_loc_plan.reads_from(
|
||||
draft_translator,
|
||||
seq_lens=draft_seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
read_extent=block_size,
|
||||
)
|
||||
if self.use_compact_draft_cache
|
||||
else kv_loc_plan
|
||||
)
|
||||
draft_plan.bind(forward_batch, draft_translator)
|
||||
|
||||
if self.selector is not None or self.lilicorr is not None:
|
||||
self._selector_sample = None
|
||||
@@ -2713,6 +2737,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
draft_token_num=int(self.block_size),
|
||||
custom_mask=custom_mask,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
batch.out_cache_loc = verify_out_cache_loc
|
||||
@@ -2877,10 +2902,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
# Consume in this step: every decode graph size shares one aux output,
|
||||
# which the next target forward overwrites (resolve_aux_hidden_states_width).
|
||||
cache_loc = self._draft_write_ids(kv_loc_plan)
|
||||
self._append_target_hidden_to_draft_kv_by_loc(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
cache_loc=verify_out_cache_loc,
|
||||
cache_loc_2d=verify_out_cache_loc_2d,
|
||||
cache_loc=cache_loc,
|
||||
cache_loc_2d=cache_loc.view(verify_out_cache_loc_2d.shape),
|
||||
positions=positions,
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
|
||||
@@ -360,7 +360,7 @@ class DraftBlockProposer:
|
||||
)
|
||||
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
|
||||
# loc to the draft's kernel-facing ids (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(idle_batch)
|
||||
self.draft_model_runner.kv_index_translator.bind_own_plan(idle_batch)
|
||||
self._fill_dp_moe_sync_metadata(idle_batch, batch)
|
||||
with torch.inference_mode():
|
||||
self.draft_model_runner.forward(idle_batch)
|
||||
@@ -435,10 +435,10 @@ class DraftBlockProposer:
|
||||
),
|
||||
global_num_token_non_padded_cpu=draft_num_tokens,
|
||||
)
|
||||
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
|
||||
# loc to the draft's kernel-facing ids (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(
|
||||
draft_forward_batch
|
||||
verify_window.kv_loc_plan.bind(
|
||||
draft_forward_batch,
|
||||
self.draft_model_runner.kv_index_translator,
|
||||
cols=slice(0, query_token_num),
|
||||
)
|
||||
self._fill_dp_moe_sync_metadata(draft_forward_batch, batch)
|
||||
graph_runner = self.draft_model_runner.decode_cuda_graph_runner
|
||||
|
||||
@@ -5,12 +5,12 @@ import torch
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
BuildCommitInjectLayout,
|
||||
build_unified_commit_inject_layout,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
|
||||
|
||||
@@ -32,6 +32,11 @@ class TargetHiddenKvInjector:
|
||||
self.verify_num_draft_tokens = verify_num_draft_tokens
|
||||
self._block_pos_offsets = block_pos_offsets
|
||||
|
||||
def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor:
|
||||
"""The plan's write ids as the draft pool indexes them: the target's
|
||||
physical ids for a fused draft, the virtual ids for a private one."""
|
||||
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
|
||||
|
||||
def inject_target_hidden(
|
||||
self,
|
||||
*,
|
||||
@@ -70,16 +75,7 @@ class TargetHiddenKvInjector:
|
||||
device=device, dtype=torch.int64, non_blocking=True
|
||||
)
|
||||
|
||||
# On a translating pool the translate writes a new tensor, so the
|
||||
# callers' locs stay virtual; on a plain pool it returns `cache_loc`
|
||||
# itself.
|
||||
translator = self.draft_model_runner.kv_index_translator
|
||||
cache_loc = translator.translate_full_attn_ids(cache_loc)
|
||||
if cache_loc_2d is not None:
|
||||
# Callers pass `cache_loc` viewed [bs, verify_len]: reshape the
|
||||
# translated copy rather than translate the same ids twice.
|
||||
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
|
||||
|
||||
# `cache_loc` arrives in the draft pool's id space (`ids_for`).
|
||||
pool = self.draft_model_runner.token_to_kv_pool
|
||||
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
|
||||
if target_hidden_is_projected:
|
||||
@@ -214,6 +210,7 @@ class TargetHiddenKvInjector:
|
||||
hidden_strided: torch.Tensor,
|
||||
commit_lens: torch.Tensor,
|
||||
bs: int,
|
||||
kv_loc_plan: KVLocPlan,
|
||||
) -> None:
|
||||
stride = self.verify_num_draft_tokens
|
||||
prefix_lens = batch.seq_lens
|
||||
@@ -252,15 +249,7 @@ class TargetHiddenKvInjector:
|
||||
return
|
||||
|
||||
positions_2d = prefix_lens.unsqueeze(1) + self._block_pos_offsets
|
||||
verify_cache_loc = assign_extend_cache_locs_func(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=self.model_runner.req_to_token_pool.req_to_token,
|
||||
start_offset=prefix_lens,
|
||||
end_offset=prefix_lens + stride,
|
||||
batch_size=bs,
|
||||
draft_token_num=stride,
|
||||
device=self.device,
|
||||
)
|
||||
verify_cache_loc = self.ids_for(kv_loc_plan)
|
||||
verify_cache_loc_2d = verify_cache_loc.view(bs, stride)
|
||||
self.inject_target_hidden(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.srt.managers.overlap_utils import (
|
||||
ResolvedConfidence,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
|
||||
from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments
|
||||
@@ -65,6 +66,9 @@ class VerifyWindow(msgspec.Struct, frozen=True):
|
||||
positions_2d: torch.Tensor
|
||||
verify_cache_loc: torch.Tensor
|
||||
verify_cache_loc_2d: torch.Tensor
|
||||
# The iteration's plan of `verify_cache_loc`: the draft block, the verify
|
||||
# and the target-hidden KV writes all take their ids from it.
|
||||
kv_loc_plan: KVLocPlan
|
||||
|
||||
|
||||
class DSparkVerifyPlanner:
|
||||
@@ -827,7 +831,10 @@ def alloc_verify_window(
|
||||
verify_num_draft_tokens: int,
|
||||
block_pos_offsets: torch.Tensor,
|
||||
model_runner,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
) -> VerifyWindow:
|
||||
"""``seq_lens_cpu``: a host bound on ``batch.seq_lens``, which sizes the
|
||||
plan's read table."""
|
||||
prefix_lens = batch.seq_lens
|
||||
verify_w = verify_num_draft_tokens
|
||||
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
|
||||
@@ -845,6 +852,13 @@ def alloc_verify_window(
|
||||
positions_2d=positions_2d,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_cache_loc_2d=verify_cache_loc_2d,
|
||||
kv_loc_plan=model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=verify_cache_loc,
|
||||
read_extent=verify_w,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -312,6 +312,7 @@ class TargetVerifyExecutor:
|
||||
custom_mask=None,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
live_seq_lens_cpu=batch.seq_lens_cpu,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
batch.out_cache_loc = verify_cache_loc
|
||||
seq_lens_cpu_backup = batch.seq_lens_cpu
|
||||
@@ -388,6 +389,7 @@ class TargetVerifyExecutor:
|
||||
hidden_strided=hidden_strided,
|
||||
commit_lens=commit_lens,
|
||||
bs=bs,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
return
|
||||
hidden = logits_output.hidden_states
|
||||
@@ -404,10 +406,11 @@ class TargetVerifyExecutor:
|
||||
state_slot = (
|
||||
batch.req_pool_indices[:bs].view(-1, 1).expand(bs, vlen).reshape(-1)
|
||||
)
|
||||
cache_loc = self.kv_injector.ids_for(verify_window.kv_loc_plan)
|
||||
self.kv_injector.inject_target_hidden(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
cache_loc=verify_window.verify_cache_loc,
|
||||
cache_loc_2d=verify_window.verify_cache_loc_2d,
|
||||
cache_loc=cache_loc,
|
||||
cache_loc_2d=cache_loc.view(verify_window.verify_cache_loc_2d.shape),
|
||||
positions=verify_window.positions_2d.reshape(-1),
|
||||
commit_lens=commit_lens,
|
||||
state_slot=state_slot,
|
||||
@@ -420,6 +423,7 @@ class TargetVerifyExecutor:
|
||||
layout: RaggedVerifyLayout,
|
||||
ragged_window: RaggedVerifyWindow,
|
||||
sampling_info,
|
||||
kv_loc_plan,
|
||||
) -> TargetVerifyResult:
|
||||
verify_input = DFlashVerifyInput(
|
||||
draft_token=ragged_window.verify_ids,
|
||||
@@ -429,6 +433,9 @@ class TargetVerifyExecutor:
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
ragged_verify_layout=layout,
|
||||
live_seq_lens_cpu=batch.seq_lens_cpu,
|
||||
# The packed rows are a selection of the planned verify window.
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
kv_loc_cols=ragged_window.window_index,
|
||||
)
|
||||
batch.out_cache_loc = ragged_window.verify_cache_loc
|
||||
seq_lens_cpu_backup = batch.seq_lens_cpu
|
||||
@@ -461,6 +468,7 @@ class TargetVerifyExecutor:
|
||||
bs: int,
|
||||
device: str,
|
||||
sampling_info,
|
||||
verify_window: VerifyWindow,
|
||||
inject_gate: bool = False,
|
||||
) -> tuple[TargetVerifyResult, torch.Tensor]:
|
||||
ragged_window = BuildRaggedVerifyWindow.execute(
|
||||
@@ -480,6 +488,7 @@ class TargetVerifyExecutor:
|
||||
layout=layout,
|
||||
ragged_window=ragged_window,
|
||||
sampling_info=sampling_info,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
logits_output = target_verify.logits_output
|
||||
|
||||
|
||||
@@ -642,6 +642,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
|
||||
target_hidden_is_projected = (
|
||||
@@ -705,7 +706,9 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
cache_loc = batch.out_cache_loc
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
|
||||
batch_output.kv_loc_plan = None
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
if token_indices is not None:
|
||||
cache_loc = cache_loc[token_indices]
|
||||
@@ -842,6 +845,11 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
block_pos_offsets=self._block_pos_offsets,
|
||||
model_runner=self.model_runner,
|
||||
seq_lens_cpu=(
|
||||
batch.seq_lens_cpu
|
||||
if batch.seq_lens_cpu is not None
|
||||
else draft_input.nxt_kv_lens_cpu
|
||||
),
|
||||
)
|
||||
|
||||
sampling_info = batch.sampling_info
|
||||
@@ -936,6 +944,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
bs=bs,
|
||||
device=device,
|
||||
sampling_info=sampling_info,
|
||||
verify_window=verify_window,
|
||||
inject_gate=fold_eligible,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -459,7 +459,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=num_seqs,
|
||||
input_ids=None,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -485,6 +484,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL
|
||||
),
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
def run_once():
|
||||
self.draft_attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
|
||||
@@ -424,7 +424,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -448,6 +447,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
if self.buffers.dsa_seed_topk_capture is not None:
|
||||
spec_info.dsa_seed_topk_capture = self.buffers.dsa_seed_topk_capture[
|
||||
@@ -649,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
encoder_lens=None,
|
||||
out_cache_loc=buffers.out_cache_loc[:num_tokens],
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
@@ -34,6 +35,9 @@ class EagleVerifyInput(SpecInput):
|
||||
draft_probs: torch.Tensor = None
|
||||
prepared_out_cache_loc: Optional[torch.Tensor] = None
|
||||
prepared_mrope_positions: Optional[torch.Tensor] = None
|
||||
# The iteration's plan when the draft planned the verify window
|
||||
# (`prepared_out_cache_loc`); verify and draft extend take it.
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
|
||||
# Shape info for padding
|
||||
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
|
||||
@@ -90,11 +94,12 @@ class EagleVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
device = req_pool_indices.device
|
||||
batch_size = req_pool_indices.numel()
|
||||
qo_indptr = torch.arange(
|
||||
@@ -113,13 +118,13 @@ class EagleVerifyInput(SpecInput):
|
||||
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
mask_numel = (
|
||||
paged_kernel_lens_sum * self.draft_token_num
|
||||
@@ -402,10 +407,11 @@ class EagleDraftExtendInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: Optional[int],
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""Draft-extend CSR args. The lens already include the window, so
|
||||
unlike verify nothing is widened here."""
|
||||
"""Draft-extend CSR args. ``paged_kernel_lens`` already includes the
|
||||
draft-extend window."""
|
||||
device = req_pool_indices.device
|
||||
bs = self.num_correct_drafts.numel()
|
||||
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
|
||||
@@ -426,12 +432,12 @@ class EagleDraftExtendInput(SpecInput):
|
||||
paged_kernel_lens_sum, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
return kv_indices, cum_kv_seq_len, qo_indptr, None
|
||||
|
||||
@@ -607,6 +607,7 @@ def eagle_prepare_for_verify(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=verify_input.prepared_mrope_positions,
|
||||
kv_loc_plan=verify_input.kv_loc_plan,
|
||||
)
|
||||
|
||||
# Run attention backend plan and cuda graph preparation
|
||||
|
||||
@@ -55,6 +55,8 @@ if _is_cpu:
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
|
||||
@@ -121,7 +123,10 @@ def prepare_for_draft_extend(
|
||||
return_hidden_states_before_norm: bool,
|
||||
widened_out_cache_loc: Optional[torch.Tensor] = None,
|
||||
widened_positions: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""``kv_loc_plan``: the iteration's plan, whose window the draft extend
|
||||
writes again (the verify window); a widened window plans its own."""
|
||||
bs = len(batch.seq_lens)
|
||||
# Optional window widening (num_front_tokens=0 -> off): prepend that many
|
||||
# rows below the boundary. Locs/positions arrive precomputed; token/hidden
|
||||
@@ -204,6 +209,7 @@ def prepare_for_draft_extend(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=return_hidden_states_before_norm,
|
||||
extend_position_info=extend_position_info,
|
||||
kv_loc_plan=None if widen else kv_loc_plan,
|
||||
)
|
||||
# Forward sees post-write length (draft extend writes num_draft_tokens
|
||||
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
|
||||
@@ -243,18 +249,35 @@ def prepare_for_draft(
|
||||
draft_model_runner: ModelRunner,
|
||||
topk: int,
|
||||
num_steps: int,
|
||||
):
|
||||
|
||||
*,
|
||||
target_translator: Optional[KVIndexTranslator] = None,
|
||||
num_draft_tokens: Optional[int] = None,
|
||||
) -> tuple[ForwardBatch, bool, Optional[KVLocPlan]]:
|
||||
"""The draft-decode batch. With ``target_translator``, a chain draft plans
|
||||
the iteration's whole write window (the ``num_draft_tokens`` slots past
|
||||
``seq_lens`` that verify and draft extend also write) and writes its first
|
||||
``num_steps`` columns; that plan is returned, else None."""
|
||||
prepared_positions = None
|
||||
prepared_mrope_positions = None
|
||||
kv_loc_plan = None
|
||||
if not batch.forward_mode.is_idle():
|
||||
bs = len(batch.seq_lens)
|
||||
|
||||
# Assign cache locations (draft-write targets).
|
||||
page_size = batch.token_to_kv_pool_allocator.page_size
|
||||
plans_window = (
|
||||
target_translator is not None
|
||||
and topk == 1
|
||||
and num_draft_tokens is not None
|
||||
and num_draft_tokens > num_steps
|
||||
and not _is_cpu
|
||||
)
|
||||
# A chain draft's slots lead the iteration's window: gather the whole
|
||||
# window in the same launch.
|
||||
window_cols = num_draft_tokens if plans_window else num_steps
|
||||
if page_size == 1 or topk == 1:
|
||||
batch.out_cache_loc = torch.empty(
|
||||
(bs * topk * num_steps,),
|
||||
(bs * topk * window_cols,),
|
||||
dtype=torch.int64,
|
||||
device=batch.device,
|
||||
)
|
||||
@@ -285,13 +308,21 @@ def prepare_for_draft(
|
||||
batch.out_cache_loc,
|
||||
req_to_token_pool.req_to_token.shape[1],
|
||||
topk,
|
||||
num_steps,
|
||||
window_cols,
|
||||
positions=prepared_positions,
|
||||
mrope=prepared_mrope_positions,
|
||||
BS=bs,
|
||||
WRITE_POSITIONS=prepared_positions is not None,
|
||||
WRITE_MROPE=prepared_mrope_positions is not None,
|
||||
)
|
||||
if plans_window:
|
||||
kv_loc_plan = target_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=batch.seq_lens,
|
||||
seq_lens_cpu=batch.seq_lens_cpu,
|
||||
write_virtual=batch.out_cache_loc,
|
||||
read_extent=num_draft_tokens,
|
||||
)
|
||||
else:
|
||||
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
|
||||
# Reduce out_cache_loc from the page-aligned tree region down to the
|
||||
@@ -352,11 +383,13 @@ def prepare_for_draft(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=prepared_mrope_positions,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
write_cols=None if kv_loc_plan is None else slice(0, num_steps),
|
||||
)
|
||||
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
|
||||
forward_batch
|
||||
)
|
||||
return forward_batch, can_run_decode_cuda_graph
|
||||
return forward_batch, can_run_decode_cuda_graph, kv_loc_plan
|
||||
|
||||
|
||||
def build_eagle_verify_input(
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
@@ -606,7 +607,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
def draft(self, batch: ScheduleBatch, *, with_topology: bool = False):
|
||||
draft_input: EagleDraftInput = batch.spec_info
|
||||
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
|
||||
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
|
||||
draft_input,
|
||||
self.req_to_token_pool,
|
||||
batch,
|
||||
@@ -614,6 +615,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
target_translator=self.target_worker.model_runner.kv_index_translator,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
)
|
||||
if (
|
||||
can_run_decode_cuda_graph
|
||||
@@ -673,6 +676,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
tree_mask_mode=self.tree_mask_mode,
|
||||
device=self.device,
|
||||
)
|
||||
if kv_loc_plan is not None:
|
||||
# Verify and draft extend write the window the draft planned.
|
||||
verify_input.kv_loc_plan = kv_loc_plan
|
||||
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
|
||||
if with_topology:
|
||||
# PP+spec relays the tree so every stage rebuilds the same verify
|
||||
# input; the mask build needs the topology this one was built from.
|
||||
@@ -900,6 +907,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
target_hidden_states: torch.Tensor,
|
||||
next_token_ids: torch.Tensor,
|
||||
mm_input_embeds: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""
|
||||
Run draft model extend to correctly fill the KV cache.
|
||||
@@ -908,6 +916,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: The batch to run.
|
||||
target_hidden_states: Hidden states from the target model forward
|
||||
next_token_ids: Next token ids generated from the target forward.
|
||||
kv_loc_plan: The target prefill's plan; the draft writes the same
|
||||
slots.
|
||||
"""
|
||||
# Construct input_ids
|
||||
if not batch.forward_mode.is_idle():
|
||||
@@ -959,6 +969,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
forward_batch.return_logprob = False
|
||||
if mm_input_embeds is not None:
|
||||
@@ -1035,7 +1046,11 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
return buf[:num_tokens]
|
||||
|
||||
def _draft_extend_for_decode(
|
||||
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
batch_result: GenerationBatchResult,
|
||||
*,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
# Cast to int64 before entering plan stream to avoid cross-stream
|
||||
# synchronization issues with .to() inside the plan stream context.
|
||||
@@ -1093,6 +1108,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
self.cuda_graph_runner_for_draft_extend,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.plan_stream:
|
||||
@@ -1412,7 +1428,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
):
|
||||
self.draft_worker._draft_extend_for_decode(batch, batch_output)
|
||||
self.draft_worker._draft_extend_for_decode(
|
||||
batch, batch_output, kv_loc_plan=verify_input.kv_loc_plan
|
||||
)
|
||||
|
||||
if (
|
||||
get_parallel().pp_size > 1
|
||||
@@ -1468,7 +1486,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
return_kv_loc_plan=self._draft_worker is not None,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
@@ -1505,6 +1525,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
batch_output.logits_output.mm_input_embeds,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
@@ -302,7 +302,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=expanded_bs,
|
||||
input_ids=None,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -321,6 +320,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
def run_once():
|
||||
# Record the metadata rebuild against the committed target-prefix
|
||||
|
||||
@@ -365,6 +365,8 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
encoder_lens=None,
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
spec_info=None,
|
||||
)
|
||||
with self._frozen_kv_target_view(forward_batch):
|
||||
|
||||
@@ -99,6 +99,7 @@ if is_npu():
|
||||
fill_draft_extend_prepare_buffers = fill_draft_extend_prepare_buffers_native
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.speculative.multi_layer_eagle_worker_v2 import (
|
||||
MultiLayerEagleDraftWorker,
|
||||
)
|
||||
@@ -333,7 +334,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Forward batch
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -361,6 +361,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
global_num_token_non_padded_cpu=self.captured_req_width * bs,
|
||||
return_hidden_states_before_norm=True,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
return forward_batch
|
||||
|
||||
def _postprocess_forward_batch(self, forward_batch: ForwardBatch, bs: int):
|
||||
@@ -476,6 +477,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
*,
|
||||
out_cache_loc_virtual: Optional[torch.Tensor],
|
||||
kv_loc_plan: Optional[KVLocPlan],
|
||||
kv_loc_cols: Optional[Cols],
|
||||
):
|
||||
"""Init this step's attention metadata for the prepared bucket and
|
||||
replay its graph. Buffers must already be populated by the composite
|
||||
@@ -501,6 +504,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
out_cache_loc_virtual=out_cache_loc_virtual,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
kv_loc_cols=kv_loc_cols,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
if (
|
||||
@@ -550,6 +555,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
self.num_front_tokens = 0
|
||||
self.prune_draft_extend_logits = False
|
||||
self._out_cache_loc_virtual = None
|
||||
self._kv_loc_plan = None
|
||||
self._kv_loc_cols = None
|
||||
|
||||
self._init_and_capture()
|
||||
|
||||
@@ -740,11 +747,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
req_pool_indices,
|
||||
out_cache_loc,
|
||||
positions=None,
|
||||
out_cache_loc_virtual=None,
|
||||
out_cache_loc_virtual,
|
||||
kv_loc_plan,
|
||||
):
|
||||
# Staging runs before `prepare`, so the rail arrives here rather than
|
||||
# off a ForwardBatch; `prepare` overwrites it with the batch's own.
|
||||
# Staging runs before `prepare` has a ForwardBatch, so the virtual write
|
||||
# ids and the plan come in here; `prepare` replaces them with the batch's.
|
||||
self._out_cache_loc_virtual = out_cache_loc_virtual
|
||||
self._kv_loc_plan = kv_loc_plan
|
||||
self._kv_loc_cols = None
|
||||
raw_bs = req_pool_indices.shape[0]
|
||||
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
|
||||
buffers = self.buffers
|
||||
@@ -780,6 +790,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
out_cache_loc_virtual=self._out_cache_loc_virtual,
|
||||
kv_loc_plan=self._kv_loc_plan,
|
||||
kv_loc_cols=self._kv_loc_cols,
|
||||
)
|
||||
for backend in backends:
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
@@ -792,6 +804,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
num_tokens = raw_bs * self.captured_req_width
|
||||
|
||||
self._out_cache_loc_virtual = forward_batch.out_cache_loc_virtual
|
||||
self._kv_loc_plan = forward_batch.kv_loc_plan
|
||||
self._kv_loc_cols = forward_batch.kv_loc_cols
|
||||
|
||||
# Bucketize to a captured batch size (padding the tail).
|
||||
if self.require_mlp_tp_gather:
|
||||
@@ -900,6 +914,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
self._replay_spec_info,
|
||||
self.seq_lens_cpu,
|
||||
out_cache_loc_virtual=self._out_cache_loc_virtual,
|
||||
kv_loc_plan=self._kv_loc_plan,
|
||||
kv_loc_cols=self._kv_loc_cols,
|
||||
)
|
||||
raw_bs = self.raw_bs
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
|
||||
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, List, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -107,6 +107,7 @@ _is_cpu = is_cpu()
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner, ModelRunnerOutput
|
||||
|
||||
|
||||
@@ -460,7 +461,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
def draft(self, batch: ScheduleBatch):
|
||||
draft_input: EagleDraftInput = batch.spec_info
|
||||
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
|
||||
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
|
||||
draft_input,
|
||||
self.req_to_token_pool,
|
||||
batch,
|
||||
@@ -468,12 +469,14 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner_list[0],
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
target_translator=self.target_worker.model_runner.kv_index_translator,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
)
|
||||
|
||||
# Run draft
|
||||
parent_list, top_scores_index, draft_tokens = self.draft_forward(forward_batch)
|
||||
|
||||
return build_eagle_verify_input(
|
||||
verify_input = build_eagle_verify_input(
|
||||
batch,
|
||||
draft_input,
|
||||
parent_list,
|
||||
@@ -487,6 +490,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
tree_mask_mode=self.tree_mask_mode,
|
||||
device=self.device,
|
||||
)
|
||||
if kv_loc_plan is not None:
|
||||
# Verify and draft extend write the window the draft planned.
|
||||
verify_input.kv_loc_plan = kv_loc_plan
|
||||
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
|
||||
return verify_input
|
||||
|
||||
def draft_forward(self, forward_batch: ForwardBatch):
|
||||
# Parse args
|
||||
@@ -596,6 +604,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: ScheduleBatch,
|
||||
target_hidden_states: torch.Tensor,
|
||||
next_token_ids: torch.Tensor,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""
|
||||
Run draft model extend to correctly fill the KV cache.
|
||||
@@ -604,6 +613,8 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: The batch to run.
|
||||
target_hidden_states: Hidden states from the target model forward
|
||||
next_token_ids: Next token ids generated from the target forward.
|
||||
kv_loc_plan: The target prefill's plan; the draft writes the same
|
||||
slots.
|
||||
"""
|
||||
# The draft embed clamps unconditionally (to tolerate multimodal pad
|
||||
# sentinels), so probe next_token_ids here first -- otherwise a corrupted id
|
||||
@@ -641,6 +652,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner_list[0],
|
||||
capture_hidden_mode=draft_capture_hidden_mode,
|
||||
return_hidden_states_before_norm=True,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
self._apply_deferred_mamba_init_to_draft_pools(forward_batch)
|
||||
@@ -766,14 +778,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
draft_token_num=self.speculative_num_draft_tokens,
|
||||
device=batch.device,
|
||||
)
|
||||
seq_lens = batch.seq_lens + self.speculative_num_draft_tokens
|
||||
runner.stage_shared_reads(
|
||||
seq_lens=batch.seq_lens + self.speculative_num_draft_tokens,
|
||||
seq_lens=seq_lens,
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
out_cache_loc=locs,
|
||||
positions=positions,
|
||||
# Both loc sources read req_to_token untranslated, so `locs` is the
|
||||
# PRE-translate write loc the capture rail needs.
|
||||
# Both loc sources read req_to_token untranslated: `locs` is virtual.
|
||||
out_cache_loc_virtual=locs,
|
||||
kv_loc_plan=self.draft_runner_list[0].kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=None,
|
||||
write_virtual=locs,
|
||||
),
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -783,6 +801,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch_result: GenerationBatchResult,
|
||||
*,
|
||||
staged: bool = False,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
# Batch 2: Draft extend
|
||||
draft_extend_input = EagleDraftExtendInput(
|
||||
@@ -826,6 +845,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
return_hidden_states_before_norm=True,
|
||||
widened_out_cache_loc=boundary_kv_locs,
|
||||
widened_positions=boundary_kv_positions,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.plan_stream:
|
||||
@@ -1091,7 +1111,9 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
@@ -1106,6 +1128,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
else:
|
||||
@@ -1134,7 +1157,10 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
if on_publish is not None:
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
self.draft_worker._draft_extend_for_decode(
|
||||
batch, batch_output, staged=staged
|
||||
batch,
|
||||
batch_output,
|
||||
staged=staged,
|
||||
kv_loc_plan=verify_input.kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
|
||||
|
||||
@@ -63,11 +64,12 @@ class NgramVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
|
||||
@@ -83,13 +85,13 @@ class NgramVerifyInput(SpecInput):
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.
|
||||
|
||||
@@ -297,7 +297,7 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
draft_token_num=self.forward_width,
|
||||
)
|
||||
|
||||
return ForwardBatch(
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.TARGET_VERIFY,
|
||||
batch_size=len(prefix_lens),
|
||||
input_ids=input_ids,
|
||||
@@ -312,6 +312,8 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_own_plan(forward_batch)
|
||||
return forward_batch
|
||||
|
||||
def _run_target_block(
|
||||
self,
|
||||
|
||||
@@ -761,6 +761,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
batch.extend_prefix_lens = torch.tensor(
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
@@ -419,6 +420,13 @@ class MockDSV4ModelRunner:
|
||||
identity = torch.arange(swa_size, dtype=torch.int64, device=device)
|
||||
self.token_to_kv_pool.register_mapping(identity)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.kv_index_translator = KVIndexTranslator(
|
||||
req_to_token=self.req_to_token_pool.req_to_token,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
page_size=case.page_size,
|
||||
device=device,
|
||||
)
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
self.init_new_workspace = False
|
||||
@@ -661,6 +669,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
# extend_* fields are only populated for extend-shaped modes. DECODE leaves
|
||||
# them at their defaults; the flash_mla path reads metadata directly from
|
||||
# DSV4AttnMetadata so the extend fields are unused for the compress_ratio=0
|
||||
|
||||
@@ -322,6 +322,7 @@ class MockGDNModelRunner(ModelRunner):
|
||||
page_size=case.page_size,
|
||||
get_kvcache=lambda: self.token_to_kv_pool,
|
||||
)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -329,7 +330,6 @@ class MockGDNModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -562,6 +562,8 @@ def _make_forward_batch(
|
||||
else None
|
||||
),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -318,6 +318,7 @@ class MockKDAModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -325,7 +326,6 @@ class MockKDAModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -552,6 +552,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -327,6 +327,7 @@ class MockLightningModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -334,7 +335,6 @@ class MockLightningModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -523,6 +523,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -460,7 +461,6 @@ class MockMamba2ModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -653,6 +653,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -307,6 +307,7 @@ class MockMLAModelRunner(ModelRunner):
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -314,7 +315,6 @@ class MockMLAModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = True
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
# Runner-mode helpers mutate speculative graph sizes after construction.
|
||||
self.graph_shared_output = GraphSharedOutput(
|
||||
@@ -693,6 +693,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -297,6 +297,8 @@ def _init_cuda_graph_replay_metadata(backend, capture_batch_size: int, batch):
|
||||
encoder_lens=batch.encoder_lens,
|
||||
extend_seq_lens=getattr(batch, "extend_seq_lens", None),
|
||||
out_cache_loc=getattr(batch, "out_cache_loc", None),
|
||||
kv_loc_plan=batch.kv_loc_plan,
|
||||
kv_loc_cols=batch.kv_loc_cols,
|
||||
spec_info=batch.spec_info,
|
||||
)
|
||||
backend.init_forward_metadata_out_graph(fb_view)
|
||||
|
||||
@@ -510,6 +510,7 @@ def run_eagle_draft_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
|
||||
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
|
||||
expected = _run_eagle_draft_eager(
|
||||
eager_worker,
|
||||
eager_batch,
|
||||
@@ -526,6 +527,7 @@ def run_eagle_draft_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
|
||||
graph_runner = _capture_eagle_draft_graph_runner(
|
||||
graph_worker,
|
||||
graph_backend,
|
||||
@@ -565,6 +567,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
|
||||
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
|
||||
expected = _run_frozen_kv_mtp_eager(eager_worker, eager_batch)
|
||||
|
||||
graph_fixture, graph_worker, _ = _build_frozen_kv_mtp_fixture(
|
||||
@@ -576,6 +579,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
|
||||
graph_runner = _capture_frozen_kv_mtp_graph_runner(graph_worker)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user