[Refactor] Translate each KV loc once per iteration (#42753)
This commit is contained in:
@@ -30,10 +30,11 @@ Two delivery forms over that one formula:
|
||||
which the pool bounds, where a page table's width is bounded
|
||||
only by `max_context_len`.
|
||||
|
||||
PREFIX-ONLY per row: nothing past the row's live prefix is written, so a
|
||||
caller-owned buffer keeps what it had there -- which is what lets a captured
|
||||
cuda-graph buffer be refreshed in place. Readers bound themselves by
|
||||
`cache_seqlens` and never look past the prefix.
|
||||
PREFIX-ONLY per row by default: nothing past the row's live prefix is written,
|
||||
so a caller-owned buffer keeps what it had there -- which is what lets a
|
||||
captured cuda-graph buffer be refreshed in place. Readers bound themselves by
|
||||
`cache_seqlens` and never look past the prefix. A fresh (`torch.empty`) table
|
||||
asks for the sink past the prefix instead (`zero_tail`).
|
||||
|
||||
A `-1` in `req_to_token` and a freed (`-1`) v2p row both clamp to entry 0, the
|
||||
reserved padding slot, so a kernel dereferences padding, not a wild address.
|
||||
@@ -60,6 +61,56 @@ _TARGET_BLOCKS = 1024
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_translate_items(
|
||||
row_in, # this row of `req_to_token` -- VIRTUAL token ids
|
||||
row_out, # where this row's items land
|
||||
v2p_ptr, # [num_pages + 1] int64 -- virtual->physical page table
|
||||
live, # scalar: the row reads `req_to_token`; else every item is the sink
|
||||
kv_start, # first token of the row's window
|
||||
n_items, # live items in the row
|
||||
n_written, # items stored: `n_items`, or a wider zeroed row
|
||||
item_begin, # this program's first item
|
||||
item_stride, # items one program advances per loop trip
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
EMIT_PER_TOKEN: tl.constexpr,
|
||||
OUT_INT64: tl.constexpr,
|
||||
ZERO_TAIL: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
for start in range(item_begin, n_written, item_stride):
|
||||
item = start + tl.arange(0, BLOCK)
|
||||
mask = item < n_items
|
||||
load_mask = mask & live
|
||||
pos = kv_start + item
|
||||
if EMIT_PER_TOKEN:
|
||||
page = pos // PAGE_SIZE
|
||||
else:
|
||||
page = pos
|
||||
tok = tl.load(
|
||||
row_in + page.to(tl.int64) * PAGE_SIZE, mask=load_mask, other=0
|
||||
).to(tl.int64)
|
||||
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
|
||||
# -1 at ps == 1, which would read one element BEFORE `v2p`.
|
||||
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
|
||||
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=load_mask, other=0), 0)
|
||||
if EMIT_PER_TOKEN:
|
||||
value = entry * PAGE_SIZE + pos % PAGE_SIZE
|
||||
else:
|
||||
value = entry
|
||||
store_mask = mask
|
||||
if ZERO_TAIL:
|
||||
value = tl.where(mask, value, 0)
|
||||
store_mask = item < n_written
|
||||
if OUT_INT64:
|
||||
tl.store(row_out + item, value, mask=store_mask)
|
||||
else:
|
||||
tl.store(row_out + item, value.to(tl.int32), mask=store_mask)
|
||||
|
||||
|
||||
# The per-call widths and lengths stay unspecialized: Triton would otherwise
|
||||
# compile a new variant (`== 1`, `% 16`) mid-serving each time a batch's shape
|
||||
# hits a combination it has not seen.
|
||||
@triton.jit(do_not_specialize=["out_stride", "seq_len_delta", "max_row_items"])
|
||||
def build_kv_read_indices_kernel(
|
||||
req_to_token_ptr, # in: [max_reqs, max_context] -- VIRTUAL token ids
|
||||
req_pool_indices_ptr, # in: [bs] -- req_to_token row per batch lane
|
||||
@@ -76,6 +127,7 @@ def build_kv_read_indices_kernel(
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
EMIT_PER_TOKEN: tl.constexpr,
|
||||
OUT_INT64: tl.constexpr,
|
||||
ZERO_TAIL: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
@@ -98,30 +150,24 @@ def build_kv_read_indices_kernel(
|
||||
# A widened window can outrun a fixed-width row (a verify near the
|
||||
# context limit); writing past it would land in the next row.
|
||||
n_items = tl.minimum(n_items, max_row_items)
|
||||
|
||||
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
|
||||
item = start + tl.arange(0, BLOCK)
|
||||
mask = item < n_items
|
||||
pos = kv_start + item
|
||||
if EMIT_PER_TOKEN:
|
||||
page = pos // PAGE_SIZE
|
||||
else:
|
||||
page = pos
|
||||
tok = tl.load(row_in + page.to(tl.int64) * PAGE_SIZE, mask=mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
|
||||
# -1 at ps == 1, which would read one element BEFORE `v2p`.
|
||||
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
|
||||
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=mask, other=0), 0)
|
||||
if EMIT_PER_TOKEN:
|
||||
value = entry * PAGE_SIZE + pos % PAGE_SIZE
|
||||
else:
|
||||
value = entry
|
||||
if OUT_INT64:
|
||||
tl.store(row_out + item, value, mask=mask)
|
||||
else:
|
||||
tl.store(row_out + item, value.to(tl.int32), mask=mask)
|
||||
# A fresh (unzeroed) table also takes the sink past the live prefix.
|
||||
n_written = max_row_items if ZERO_TAIL else n_items
|
||||
_gather_translate_items(
|
||||
row_in,
|
||||
row_out,
|
||||
v2p_ptr,
|
||||
True,
|
||||
kv_start,
|
||||
n_items,
|
||||
n_written,
|
||||
tl.program_id(1) * BLOCK,
|
||||
item_stride,
|
||||
PAGE_SIZE=PAGE_SIZE,
|
||||
EMIT_PER_TOKEN=EMIT_PER_TOKEN,
|
||||
OUT_INT64=OUT_INT64,
|
||||
ZERO_TAIL=ZERO_TAIL,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
|
||||
|
||||
def _launch(
|
||||
@@ -138,6 +184,7 @@ def _launch(
|
||||
kv_start_idx: Optional[torch.Tensor],
|
||||
emit_per_token: bool,
|
||||
seq_len_delta: int = 0,
|
||||
zero_tail: bool = False,
|
||||
) -> None:
|
||||
bs = int(req_pool_indices.numel())
|
||||
item_programs = min(
|
||||
@@ -159,6 +206,7 @@ def _launch(
|
||||
PAGE_SIZE=page_size,
|
||||
EMIT_PER_TOKEN=emit_per_token,
|
||||
OUT_INT64=out.dtype == torch.int64,
|
||||
ZERO_TAIL=zero_tail,
|
||||
BLOCK=_BLOCK_ITEMS,
|
||||
num_warps=_NUM_WARPS,
|
||||
)
|
||||
@@ -188,17 +236,18 @@ def build_kv_read_table(
|
||||
max_pages: int,
|
||||
out: torch.Tensor,
|
||||
seq_len_delta: int = 0,
|
||||
zero_tail: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Fill ``out``'s live prefix with PAGE TABLE entries.
|
||||
|
||||
``out`` is caller-owned (fresh zeros for the eager path, the module's
|
||||
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
|
||||
region's live prefix is written -- never rebound, never tail-cleared.
|
||||
``out`` is caller-owned and only its ``[:bs, :max_pages]`` region is
|
||||
written: each row's live prefix, and with ``zero_tail`` the sink (0) past
|
||||
it up to ``max_pages`` -- what a fresh ``torch.empty`` table needs, in the
|
||||
same launch. Never rebound.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). Every row stops at
|
||||
``max_pages``: a widened window that outruns the row is cut there rather
|
||||
than spilling into the next row, on CUDA as on CPU.
|
||||
``seq_len_delta`` widens every row's live prefix (a verify reads its draft
|
||||
tokens' KV back from the pool); a widened row is cut at ``max_pages``
|
||||
rather than spilling into the next row.
|
||||
"""
|
||||
bs = int(req_pool_indices.numel())
|
||||
assert out.dtype == torch.int32, (
|
||||
@@ -228,6 +277,8 @@ def build_kv_read_table(
|
||||
v2p=v2p,
|
||||
page_size=page_size,
|
||||
).to(torch.int32)
|
||||
if zero_tail:
|
||||
out[b, live:max_pages] = 0
|
||||
return out
|
||||
|
||||
_launch(
|
||||
@@ -243,6 +294,7 @@ def build_kv_read_table(
|
||||
kv_start_idx=None,
|
||||
emit_per_token=False,
|
||||
seq_len_delta=seq_len_delta,
|
||||
zero_tail=zero_tail,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -89,6 +89,29 @@ def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
|
||||
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _load_token_ids(
|
||||
row_ptr,
|
||||
pos,
|
||||
mask,
|
||||
v2p,
|
||||
page_size: tl.constexpr,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr,
|
||||
TRANSLATE: tl.constexpr,
|
||||
):
|
||||
"""Token ids at positions ``pos`` of one read-source row: the row's own
|
||||
entries when token-granular, ``entry * ps + pos % ps`` when page-granular;
|
||||
with ``TRANSLATE``, virtual ids mapped through ``v2p`` on the way out."""
|
||||
if ENTRY_PAGE_SIZE == 1:
|
||||
ids = tl.load(row_ptr + pos, mask=mask)
|
||||
else:
|
||||
entry = tl.load(row_ptr + pos // ENTRY_PAGE_SIZE, mask=mask, other=0)
|
||||
ids = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
|
||||
if TRANSLATE:
|
||||
ids = _translate_token_ids(ids, v2p, mask, page_size)
|
||||
return ids
|
||||
|
||||
|
||||
@triton.jit
|
||||
def generate_draft_decode_kv_indices(
|
||||
req_pool_indices,
|
||||
@@ -97,8 +120,9 @@ def generate_draft_decode_kv_indices(
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
positions,
|
||||
v2p,
|
||||
pool_len: tl.constexpr,
|
||||
# Runtime, not constexpr: the unified pool's read table is as wide as the
|
||||
# iteration's longest row, so a constexpr stride would recompile per width.
|
||||
row_stride,
|
||||
kv_indices_stride: tl.constexpr,
|
||||
kv_indptr_stride: tl.constexpr,
|
||||
bs_upper: tl.constexpr,
|
||||
@@ -108,6 +132,9 @@ def generate_draft_decode_kv_indices(
|
||||
window_size: tl.constexpr = 0,
|
||||
sink_size: tl.constexpr = 0,
|
||||
NUM_STEPS: tl.constexpr = 0,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr = 1,
|
||||
# The source's page table (`KVIndexTable.v2p`), applied with TRANSLATE.
|
||||
v2p=None,
|
||||
TRANSLATE: tl.constexpr = False,
|
||||
):
|
||||
# window_size > 0 restricts the draft (not the target) to sink_size prefix
|
||||
@@ -151,7 +178,7 @@ def generate_draft_decode_kv_indices(
|
||||
# Update kv_indices
|
||||
kv_offset = cum_seq_len * topk + bid * iters * topk + topk_id * (seq_len_w + iters)
|
||||
kv_ptr = kv_indices + kv_offset
|
||||
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * pool_len
|
||||
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
|
||||
|
||||
num_loop = tl.cdiv(seq_len_w, BLOCK_SIZE)
|
||||
if NUM_STEPS != 0 and blk >= num_loop and blk > 0:
|
||||
@@ -165,9 +192,9 @@ def generate_draft_decode_kv_indices(
|
||||
copy_offset,
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
data = _load_token_ids(
|
||||
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
|
||||
)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
copy_offset += BLOCK_SIZE
|
||||
else:
|
||||
@@ -179,9 +206,9 @@ def generate_draft_decode_kv_indices(
|
||||
copy_offset,
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
data = _load_token_ids(
|
||||
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
|
||||
)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
|
||||
# Extension entries and kv_indptr belong to token block 0 alone; other
|
||||
@@ -189,12 +216,14 @@ def generate_draft_decode_kv_indices(
|
||||
if blk == 0:
|
||||
extend_offset = tl.arange(0, iter_upper)
|
||||
if page_size == 1 or topk == 1:
|
||||
extend_data = tl.load(
|
||||
token_pool_ptr
|
||||
+ seq_len
|
||||
+ topk_id * num_steps
|
||||
+ tl.arange(0, iter_upper),
|
||||
mask=extend_offset < iters,
|
||||
extend_data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
seq_len + topk_id * num_steps + tl.arange(0, iter_upper),
|
||||
extend_offset < iters,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
else:
|
||||
prefix_len = seq_len
|
||||
@@ -208,14 +237,14 @@ def generate_draft_decode_kv_indices(
|
||||
+ topk_id * num_new_pages_per_topk * page_size
|
||||
+ last_page_len
|
||||
)
|
||||
extend_data = tl.load(
|
||||
token_pool_ptr + start + extend_offset,
|
||||
mask=extend_offset < iters,
|
||||
)
|
||||
|
||||
if TRANSLATE:
|
||||
extend_data = _translate_token_ids(
|
||||
extend_data, v2p, extend_offset < iters, page_size
|
||||
extend_data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
start + extend_offset,
|
||||
extend_offset < iters,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
|
||||
tl.store(
|
||||
|
||||
@@ -20,6 +20,9 @@ class RaggedVerifyWindow(msgspec.Struct, frozen=True):
|
||||
positions: torch.Tensor
|
||||
verify_cache_loc: torch.Tensor
|
||||
verify_ids: torch.Tensor
|
||||
# Each packed lane's flat position in the `[bs, verify_num_draft_tokens]`
|
||||
# verify window, -1 on a padded lane.
|
||||
window_index: torch.Tensor
|
||||
|
||||
|
||||
class BuildRaggedVerifyWindow:
|
||||
@@ -129,6 +132,9 @@ def build_ragged_verify_window(
|
||||
positions=positions,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_ids=verify_ids,
|
||||
window_index=torch.where(
|
||||
valid, safe_req.to(torch.int64) * verify_num_draft_tokens + within, -1
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -176,7 +182,7 @@ def build_ragged_verify_window_triton(
|
||||
verify_lens = layout.verify_lens.to(device=device, dtype=torch.int32)
|
||||
padded_total = layout.graph_num_tokens
|
||||
|
||||
req_id, within, _valid = compact_row_index_triton(
|
||||
req_id, within, valid = compact_row_index_triton(
|
||||
verify_lens=verify_lens, padded_total=padded_total, device=device
|
||||
)
|
||||
real_cache_loc = assign_extend_cache_locs_func(
|
||||
@@ -218,6 +224,11 @@ def build_ragged_verify_window_triton(
|
||||
positions=positions,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_ids=verify_ids,
|
||||
window_index=torch.where(
|
||||
valid,
|
||||
req_id.clamp(max=bs - 1).to(torch.int64) * verify_num_draft_tokens + within,
|
||||
-1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -499,10 +499,9 @@ def _assert_spec_verify_backends(
|
||||
) -> None:
|
||||
"""Refuse target backends whose verify id rails are not translation-audited.
|
||||
|
||||
Only the target's prefill/decode pair is checked here. A draft that fuses
|
||||
into the target's pages must run on the translated rails; the fused-draft
|
||||
decision enforces that by declining fusion otherwise, and a private-pool
|
||||
draft indexes its own pool by virtual id."""
|
||||
Only the target's prefill/decode pair is checked here: fusion declines for
|
||||
a draft off the translated rails, and a private-pool draft indexes its own
|
||||
pool by virtual id."""
|
||||
if allowed is None:
|
||||
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
|
||||
dcp_note = ""
|
||||
@@ -591,8 +590,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"DFLASH",
|
||||
), (
|
||||
"--enable-unified-memory only supports --speculative-algorithm "
|
||||
"DSPARK (chain draft), DFLASH (fused block draft), and "
|
||||
"EAGLE/EAGLE3 (fused draft KV); other speculative algorithms are "
|
||||
"DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are "
|
||||
"not yet audited for the unified pool's virtual-to-physical loc "
|
||||
"translation. Got "
|
||||
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
|
||||
@@ -627,9 +625,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
f"spec-verify-audited attention backends {sorted(eagle_allowed)} "
|
||||
f"(got {sorted(eagle_backends, key=str)})."
|
||||
)
|
||||
# An unset draft backend inherits the target's pair. A fused draft is
|
||||
# MHA-shaped on every host, and these rails also serve a draft that
|
||||
# keeps an MLA pool of its own.
|
||||
# An unset draft backend inherits the target's pair.
|
||||
draft_backend = cfg.speculative_draft_attention_backend
|
||||
draft_backends = {draft_backend} if draft_backend else eagle_backends
|
||||
assert draft_backends <= TRANSLATED_MHA_RAILS, (
|
||||
@@ -649,9 +645,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
)
|
||||
assert not cfg.enable_two_batch_overlap, (
|
||||
"--enable-unified-memory does not support --enable-two-batch-overlap: "
|
||||
"TBO's replay split hands each child a view without the pre-translate "
|
||||
"write loc, so a captured decode replay raises. "
|
||||
"TODO(ch-wan): carry out_cache_loc_virtual into the child view."
|
||||
"the iteration's read tables are indexed by the whole batch's rows, and "
|
||||
"a TBO child's readers do not offset into them."
|
||||
)
|
||||
assert not cfg.enable_lmcache, (
|
||||
"--enable-unified-memory is not yet compatible with --enable-lmcache: "
|
||||
@@ -679,8 +674,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
), (
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
|
||||
"--disaggregation-decode-retraction-backup=host_pool: the backup builds "
|
||||
"the draft's host pool off a device pool of its own, and an EAGLE draft "
|
||||
"on the unified pool lives fused in the target's pages, without one."
|
||||
"the draft's host pool off the draft's own device pool, which a fused "
|
||||
"EAGLE draft does not have."
|
||||
)
|
||||
assert not (
|
||||
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
|
||||
|
||||
@@ -698,6 +698,14 @@ class TboForwardBatchPreparer:
|
||||
start_token_index:end_token_index
|
||||
]
|
||||
output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical
|
||||
# Unified memory refuses two-batch overlap, so the plan's reads stay in
|
||||
# `req_to_token`, which each child indexes at its own rows; each child
|
||||
# writes its own tokens of the plan's window.
|
||||
output_dict["kv_loc_plan"] = batch.kv_loc_plan
|
||||
if batch.kv_loc_plan is not None:
|
||||
output_dict["kv_loc_cols"] = batch.kv_loc_plan.cols_slice(
|
||||
batch.kv_loc_cols, slice(start_token_index, end_token_index)
|
||||
)
|
||||
|
||||
attention_tp_size = get_parallel().attn_tp_size
|
||||
_tbo_padded_len = (
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.srt.layers.attention.triton_backend import (
|
||||
ForwardMetadata,
|
||||
update_sliding_window_buffer,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
get_capture_attention_variant,
|
||||
@@ -202,12 +203,10 @@ class DllmFlashAttention:
|
||||
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.use_sliding_window_kv_pool
|
||||
else None,
|
||||
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.kv_index_translator.is_translating
|
||||
else None,
|
||||
)
|
||||
metadata = self._prefill_graph_metadata[key]
|
||||
backend._fill_kv_indptr_and_indices(
|
||||
batch.kv_loc_plan,
|
||||
bs,
|
||||
batch.extend_prefix_lens,
|
||||
batch.req_pool_indices,
|
||||
@@ -220,6 +219,7 @@ class DllmFlashAttention:
|
||||
_, _, _, offsets = update_sliding_window_buffer(
|
||||
metadata.window_kv_indptr,
|
||||
backend.kv_index_translator,
|
||||
batch.kv_loc_plan,
|
||||
batch.req_pool_indices,
|
||||
backend.sliding_window_size,
|
||||
batch.extend_prefix_lens,
|
||||
@@ -231,15 +231,7 @@ class DllmFlashAttention:
|
||||
metadata.window_kv_offsets.copy_(offsets)
|
||||
if metadata.swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc.copy_(
|
||||
backend.kv_index_translator.sliding_window_write_loc_for(
|
||||
batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
if metadata.out_cache_loc_full_physical is not None:
|
||||
backend.kv_index_translator.fill_capture_write_loc(
|
||||
out=metadata.out_cache_loc_full_physical,
|
||||
forward_batch=batch,
|
||||
width=metadata.out_cache_loc_full_physical.numel(),
|
||||
backend.kv_index_translator.write_ids(batch, IdSpaceKind.SLIDING_WINDOW)
|
||||
)
|
||||
backend.forward_metadata = metadata
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -1951,6 +1952,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
paged_kernel_lens=forward_batch.seq_lens,
|
||||
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
)
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -2150,6 +2152,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
max(forward_batch.extend_seq_lens_cpu),
|
||||
forward_batch.seq_lens_cpu.max().item(),
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
|
||||
max_q_len = self.mla_indices_updater_prefill.max_q_len
|
||||
@@ -2190,6 +2193,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
prefix_lens,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.use_sliding_window_kv_pool:
|
||||
@@ -4454,6 +4458,8 @@ class AiterIndicesUpdaterPrefill:
|
||||
prefix_lens: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -4466,6 +4472,8 @@ class AiterIndicesUpdaterPrefill:
|
||||
prefix_lens: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
kv_start_idx = None
|
||||
kv_indptr = self.kv_indptr
|
||||
@@ -4515,6 +4523,7 @@ class AiterIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4547,6 +4556,8 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
max_q_len: int,
|
||||
max_kv_len: int,
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -4560,6 +4571,8 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
max_q_len: int,
|
||||
max_kv_len: int,
|
||||
spec_info: Optional[SpecInput],
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
bs = len(req_pool_indices)
|
||||
|
||||
@@ -4594,6 +4607,7 @@ class AiterMlaIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=kv_lens,
|
||||
paged_kernel_lens_sum=kv_lens_sum,
|
||||
translator=self.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4663,18 +4677,21 @@ class AiterMultiStepDraftBackend:
|
||||
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
|
||||
else 1
|
||||
)
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
triton.next_power_of_2(num_seqs),
|
||||
@@ -4684,7 +4701,9 @@ class AiterMultiStepDraftBackend:
|
||||
# A single token block is the historical launch; NUM_STEPS=0 keeps
|
||||
# its 128-wide program instead of the token-block specialization.
|
||||
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_flashinfer_available
|
||||
|
||||
@@ -176,7 +177,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
query = self._set_kv_and_concat_q_fp8_fused(
|
||||
layer=layer,
|
||||
@@ -208,7 +209,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
|
||||
assert k is not None and k_rope is not None
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
self._kv_write_loc(forward_batch),
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
k,
|
||||
k_rope,
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver
|
||||
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -54,6 +54,7 @@ from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.srt.utils.common import get_device_capability
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -466,10 +467,9 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
return
|
||||
if self.kv_index_translator.is_translating:
|
||||
# The draft-extend runners record this call unconditionally, but a
|
||||
# translating backend rebuilds these tables out of graph on every
|
||||
# replay; a captured raw req_to_token gather would then overwrite
|
||||
# them with virtual ids.
|
||||
# The draft-extend runners record this call unconditionally; a
|
||||
# translating backend rebuilds these tables out of graph, and a
|
||||
# captured req_to_token gather would overwrite them with virtual ids.
|
||||
return
|
||||
bs = forward_batch.batch_size
|
||||
metadata = self.draft_extend_metadata[bs]
|
||||
@@ -543,7 +543,11 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
self._init_full_cg_decode_metadata(forward_batch, in_capture)
|
||||
|
||||
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
|
||||
locations = getattr(forward_batch, "out_cache_loc", None)
|
||||
"""This step's sliding-window write ids: the plan's, sliced per
|
||||
draft step the way `out_cache_loc` is."""
|
||||
locations = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
if (
|
||||
locations is not None
|
||||
and forward_batch.forward_mode.is_decode_or_idle()
|
||||
@@ -586,11 +590,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if in_capture:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
|
||||
|
||||
if in_capture:
|
||||
num_tokens = forward_batch.positions.numel()
|
||||
@@ -627,6 +627,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
return
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -678,6 +679,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# A stale non-None seq_lens_cpu buffer would under-size max_seq_pages
|
||||
# (stale page-table rows -> OOB); force None under sync-free.
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -755,22 +757,17 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
translating = self.kv_index_translator.is_translating
|
||||
max_seq_len_k = int(forward_batch.seq_lens_cpu[:bs].max().item())
|
||||
if translating:
|
||||
# Unified pool: the block table is a TRANSLATED page table, built
|
||||
# straight into these capture-stable buffers from the LIVE v2p, so
|
||||
# a page relocated by compaction since capture is picked up. Same
|
||||
# substitution the eager extend branch makes in its `_unified_read`
|
||||
# fixup; `build_index_table` emits page-granular physical ids
|
||||
# directly, so there is no `// page_size` to undo.
|
||||
self.kv_index_translator.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices[:bs],
|
||||
seq_lens=forward_batch.seq_lens[:bs],
|
||||
into=KVReadTables(
|
||||
full=m.page_table,
|
||||
sliding_window=(
|
||||
m.swa_page_table if self.use_sliding_window_kv_pool else None
|
||||
),
|
||||
),
|
||||
# Unified pool: the plan's page table, physical page ids from this
|
||||
# iteration's v2p, copied into these capture-stable buffers.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=m.page_table
|
||||
)
|
||||
if self.use_sliding_window_kv_pool:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
out=m.swa_page_table,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
elif max_seq_len_k > 0:
|
||||
# Build the block table like the eager extend branch: take every
|
||||
# page_size-th token slot from req_to_token and divide by page_size.
|
||||
@@ -797,22 +794,15 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
self.full_cg_prefill_swa_out_cache_loc.shape[0],
|
||||
"full-CG prefill SWA write-location buffer",
|
||||
)
|
||||
# Under the unified pool `out_cache_loc` was rebound to FULL-side
|
||||
# physical ids at ForwardBatch construction, so the full->swa map
|
||||
# cannot be re-run on it: a full physical page number is also a
|
||||
# valid virtual page of the swa v2p, so the map would silently
|
||||
# resolve it to an unrelated token's swa slot. Phase 2 of the write
|
||||
# contract derives the swa loc from them instead.
|
||||
swa_write_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
if in_capture:
|
||||
# The runner-built capture batch writes the page-0 sink.
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].zero_()
|
||||
else:
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
if translating
|
||||
else self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(swa_write_loc)
|
||||
# Captured kernels read the full bucket. Route its inactive tail to
|
||||
# SWA's zero dummy slot to prevent stale writes into live slots.
|
||||
self.full_cg_prefill_swa_out_cache_loc[num_out:].zero_()
|
||||
@@ -826,17 +816,14 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def _spec_read_seq_len_delta(
|
||||
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
|
||||
) -> int:
|
||||
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
|
||||
the translated table's fill must reach ``cache_seqlens``. Takes the
|
||||
two fields rather than a ForwardBatch: the captured build slices its
|
||||
batch and has none. Now read by both builds, so eager and replay
|
||||
widen identically.
|
||||
"""Columns past ``seq_lens`` this mode's whole-sequence read covers;
|
||||
the translated table must reach ``cache_seqlens``.
|
||||
|
||||
Zero for the prefix-only shapes: normal decode/extend, the topk>1
|
||||
split (drafts read via the expand metadata), draft-extend whose lens
|
||||
already include the window, and draft-extend's idle batch. The ragged
|
||||
verify layout's per-row lens are bounded by the draft window, so its
|
||||
upper bound rides the same delta."""
|
||||
split (drafts read via the expand metadata), draft-extend (whose lens
|
||||
already include the window) and its idle batch. The ragged verify
|
||||
layout's per-row lens are bounded by the draft window, so the same
|
||||
delta bounds it."""
|
||||
if spec_info is None:
|
||||
return 0
|
||||
if forward_mode.is_target_verify() and self.topk <= 1:
|
||||
@@ -1237,27 +1224,20 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else None
|
||||
)
|
||||
if _unified_read:
|
||||
delta = self._spec_read_seq_len_delta(
|
||||
forward_batch.forward_mode, forward_batch.spec_info
|
||||
# The plan's table covers this forward's reads, a verify's and a
|
||||
# draft decode's window included; the kernels bound each row by
|
||||
# `cache_seqlens`.
|
||||
kv_view = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
|
||||
)
|
||||
if delta:
|
||||
kv_view = self.kv_index_translator.widened_index_table(
|
||||
forward_batch, seq_len_delta=delta
|
||||
)
|
||||
else:
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
metadata.page_table = kv_view.ids
|
||||
if self.use_sliding_window_kv_pool:
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
if swa_out_cache_loc is not None:
|
||||
# The swa write loc was computed from the still-VIRTUAL
|
||||
# loc at ForwardBatch construction; re-running the
|
||||
# full->swa map on the physical loc would be garbage.
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
metadata.swa_page_table = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
rows=forward_batch.batch_size,
|
||||
).ids
|
||||
metadata.swa_out_cache_loc = swa_out_cache_loc
|
||||
elif self.use_sliding_window_kv_pool:
|
||||
# FA3 requires an int32 page_table.
|
||||
metadata.swa_page_table = (
|
||||
@@ -1265,12 +1245,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.page_table
|
||||
).to(torch.int32)
|
||||
)
|
||||
if swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
metadata.swa_out_cache_loc = swa_out_cache_loc
|
||||
|
||||
# Logical-page KV sharding: capture the batch's gather plan and swap the
|
||||
# page table to scratch rows. During a sharded extend, attention reads
|
||||
@@ -2864,6 +2839,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
|
||||
def _set_decode_page_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
metadata,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -2871,9 +2847,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
) -> None:
|
||||
"""Fill `cache_seqlens_int32`, `cu_seqlens_k` and the page table(s).
|
||||
|
||||
Under the unified pool the translator writes the page tables in place,
|
||||
so the fused kernel is left with the prefix sum alone -- one pass over
|
||||
the rows instead of a translated build plus a verbatim copy of it.
|
||||
Under the unified pool the page tables are the plan's, copied in, so
|
||||
the fused kernel is left with the prefix sum alone.
|
||||
"""
|
||||
translated = self.kv_index_translator.reads_are_translated
|
||||
normal_decode_set_metadata(
|
||||
@@ -2891,17 +2866,20 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
skip_page_table=translated,
|
||||
)
|
||||
if translated:
|
||||
# Fill to `cache_seqlens_int32`, which the kernels bound their reads
|
||||
# by: a draft decode reads `seq_len_delta` past `seq_lens`.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
sliding_window_out=metadata.swa_page_table,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=metadata.cache_seqlens_int32,
|
||||
)
|
||||
# The plan's table reaches `cache_seqlens_int32`, which the kernels
|
||||
# bound their reads by: a draft decode reads past `seq_lens`.
|
||||
bs = int(req_pool_indices.numel())
|
||||
self.kv_index_translator.copy_page_table(plan, out=metadata.page_table[:bs])
|
||||
if metadata.swa_page_table is not None:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan,
|
||||
out=metadata.swa_page_table[:bs],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -2937,6 +2915,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata
|
||||
# is normal-decode-only).
|
||||
self._set_decode_page_metadata(
|
||||
plan,
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
@@ -3041,6 +3020,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else self.max_context_len
|
||||
)
|
||||
self._set_decode_page_metadata(
|
||||
plan,
|
||||
metadata,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
@@ -3092,7 +3072,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# max_seq_len_k left unset -- unread here (scheduler_metadata is
|
||||
# normal-decode-only).
|
||||
self._set_decode_page_metadata(
|
||||
metadata, req_pool_indices, verify_lens, verify_delta
|
||||
plan, metadata, req_pool_indices, verify_lens, verify_delta
|
||||
)
|
||||
else:
|
||||
# When topk > 1, we need two specific target verify metadata, and then merge states
|
||||
@@ -3225,23 +3205,19 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.max_seq_len_k + self.page_size - 1
|
||||
) // self.page_size
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
# The translator writes kernel-facing page ids straight into
|
||||
# the captured tables; `sliding_window_out` fills the swa twin
|
||||
# in the same pass.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
sliding_window_out=(
|
||||
metadata.swa_page_table
|
||||
if self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
else None
|
||||
),
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_len_delta=self._spec_read_seq_len_delta(
|
||||
forward_mode, spec_info
|
||||
),
|
||||
rows = int(req_pool_indices.numel())
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=metadata.page_table[:rows]
|
||||
)
|
||||
if (
|
||||
self.use_sliding_window_kv_pool
|
||||
and metadata.swa_page_table is not None
|
||||
):
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan,
|
||||
out=metadata.swa_page_table[:rows],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
else:
|
||||
page_indices = self.req_to_token[
|
||||
req_pool_indices[:, None],
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
@@ -61,6 +62,7 @@ from sglang.srt.utils import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -749,6 +751,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
fixed_split_size=None,
|
||||
disable_split_kv=self.disable_cuda_graph_kv_split,
|
||||
req_pool_indices=req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_target_verify():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -761,6 +764,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -773,6 +777,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=not self.use_paged,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -785,6 +790,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
elif forward_mode.is_extend():
|
||||
# Plain EXTEND under full prefill CUDA graph. plan() runs
|
||||
@@ -803,6 +809,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
|
||||
spec_info=None,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
else:
|
||||
raise ValueError("Invalid forward mode")
|
||||
@@ -863,8 +870,8 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
if in_capture:
|
||||
@@ -955,8 +962,8 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
swa_out_cache_loc = None
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
@@ -970,6 +977,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
fixed_split_size=self.decode_split_tile_size,
|
||||
disable_split_kv=False,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = DecodeMetadata(
|
||||
self.decode_wrappers, swa_out_cache_loc=swa_out_cache_loc
|
||||
@@ -985,6 +993,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
use_ragged=False,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
spec_info=forward_batch.spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrappers_verify,
|
||||
@@ -1038,6 +1047,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
cross_attention_custom_mask=forward_batch.cross_attention_custom_mask,
|
||||
extend_prefix_lens_cpu=forward_batch.extend_prefix_lens_cpu,
|
||||
custom_kv_indices=self.dq_page_table,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrappers_paged,
|
||||
@@ -1581,6 +1591,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
@@ -1597,6 +1608,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
decode_wrappers = decode_wrappers or self.decode_wrappers
|
||||
@@ -1610,6 +1622,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
seq_lens_cpu,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1624,6 +1637,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
assert self.sliding_window_size is not None
|
||||
@@ -1663,6 +1677,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
use_sliding_window_kv_pool=use_sliding_window_kv_pool,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1677,6 +1692,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Cache encoder_lens on CPU to avoid GPU→CPU transfer per call
|
||||
@@ -1703,6 +1719,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
seq_lens_cpu=kv_lens_cpu,
|
||||
fixed_split_size=fixed_split_size,
|
||||
disable_split_kv=disable_split_kv,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
)
|
||||
|
||||
@@ -1719,6 +1736,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
fixed_split_size: Optional[int] = None,
|
||||
disable_split_kv: Optional[bool] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
@@ -1726,6 +1744,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
|
||||
if spec_info is None or getattr(spec_info, "kv_indptr", None) is None:
|
||||
bs = len(paged_kernel_lens)
|
||||
kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0)
|
||||
@@ -1739,14 +1758,14 @@ class FlashInferIndicesUpdaterDecode:
|
||||
paged_kernel_lens_sum, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=use_swa_source,
|
||||
kind=read_kind,
|
||||
)
|
||||
else:
|
||||
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
|
||||
@@ -1868,6 +1887,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
# Keep the signature for type checking. It will be assigned during runtime.
|
||||
raise NotImplementedError()
|
||||
@@ -1888,6 +1909,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if use_ragged:
|
||||
assert prefix_lens is not None
|
||||
@@ -1918,6 +1941,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
multi_item_params=multi_item_params,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
custom_kv_indices=custom_kv_indices,
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def update_sliding_window(
|
||||
@@ -1936,6 +1960,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if custom_kv_indices is not None:
|
||||
raise RuntimeError(
|
||||
@@ -2031,6 +2057,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
if (wrapper_id == 0 and not use_ragged and spec_info is None)
|
||||
else -1
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def _build_swa_prefix_custom_mask(
|
||||
@@ -2090,6 +2117,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None,
|
||||
extend_prefix_lens_cpu: Optional[List[int]] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
if custom_kv_indices is not None:
|
||||
raise RuntimeError(
|
||||
@@ -2125,6 +2154,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
cross_attention_custom_mask=(
|
||||
cross_attention_custom_mask if wrapper_id == 1 else None
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
def call_begin_forward(
|
||||
@@ -2148,6 +2178,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
seq_lens_cpu: Optional[torch.Tensor] = None,
|
||||
custom_kv_indices: Optional[torch.Tensor] = None,
|
||||
window_left: int = -1,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
):
|
||||
bs = len(seq_lens)
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
@@ -2155,6 +2187,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
|
||||
if spec_info is None:
|
||||
assert prefix_lens is not None
|
||||
assert len(seq_lens) == len(req_pool_indices)
|
||||
@@ -2180,14 +2213,14 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
dtype=torch.int32,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=use_swa_source,
|
||||
kind=read_kind,
|
||||
)
|
||||
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
|
||||
qo_indptr = qo_indptr[: bs + 1]
|
||||
@@ -2202,7 +2235,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
plan=plan,
|
||||
kind=read_kind,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
)
|
||||
@@ -2213,7 +2247,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=translator,
|
||||
sliding_window=use_swa_source,
|
||||
plan=plan,
|
||||
kind=read_kind,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2392,18 +2427,21 @@ class FlashInferMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
@@ -2412,7 +2450,9 @@ class FlashInferMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
|
||||
FlashInferMlaAttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -367,6 +368,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
init_metadata_replay=False,
|
||||
spec_info=spec_info,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.decode_cuda_graph_metadata[bs] = decode_wrapper
|
||||
self.forward_metadata = DecodeMetadata(decode_wrapper)
|
||||
@@ -391,6 +393,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
raise ValueError(f"Invalid mode: {forward_mode=}")
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -408,6 +411,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
prefill_wrapper.plan = partial(fast_mla_prefill_plan, prefill_wrapper)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -425,6 +429,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
decode_wrapper=self.decode_wrapper,
|
||||
init_metadata_replay=False,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = DecodeMetadata(self.decode_wrapper)
|
||||
elif forward_batch.forward_mode.is_target_verify():
|
||||
@@ -436,6 +441,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
prefill_wrapper_paged=self.prefill_wrapper_verify,
|
||||
use_ragged=False,
|
||||
spec_info=forward_batch.spec_info,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False)
|
||||
else:
|
||||
@@ -484,6 +490,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr_cpu=kv_indptr_cpu,
|
||||
kv_len_arr_cpu=kv_len_arr_cpu,
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
)
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrapper_paged, use_ragged
|
||||
@@ -531,6 +538,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -566,6 +574,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
init_metadata_replay=True,
|
||||
spec_info=spec_info,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
plan=plan,
|
||||
**self.fast_decode_kwargs,
|
||||
)
|
||||
elif forward_mode.is_target_verify():
|
||||
@@ -616,6 +625,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
|
||||
if use_generic_fast_plan
|
||||
else None
|
||||
),
|
||||
plan=plan,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid forward mode: {forward_mode=}")
|
||||
@@ -859,6 +869,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
init_metadata_replay: bool = False,
|
||||
spec_info: Optional[SpecInput] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
**fast_decode_kwargs,
|
||||
):
|
||||
@@ -871,6 +882,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
self.kv_indptr,
|
||||
init_metadata_replay,
|
||||
spec_info,
|
||||
plan=plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
**fast_decode_kwargs,
|
||||
)
|
||||
@@ -885,6 +897,7 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
init_metadata_replay: bool = False,
|
||||
spec_info: Optional[SpecInput] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
req_pool_indices: torch.Tensor,
|
||||
**fast_decode_kwargs,
|
||||
):
|
||||
@@ -904,11 +917,11 @@ class FlashInferMLAIndicesUpdaterDecode:
|
||||
else fast_decode_kwargs["kv_indices"]
|
||||
)
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
is_translated = translator.fill_packed_read_stream(
|
||||
is_translated = translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
)
|
||||
|
||||
@@ -1006,6 +1019,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
|
||||
fast_verify_plan_kwargs: Optional[dict] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
qo_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_len_arr_cpu: Optional[torch.Tensor] = None,
|
||||
@@ -1031,6 +1045,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
spec_info,
|
||||
attn_dcp_metadata=attn_dcp_metadata,
|
||||
fast_verify_plan_kwargs=fast_verify_plan_kwargs,
|
||||
plan=plan,
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr_cpu=kv_indptr_cpu,
|
||||
kv_len_arr_cpu=kv_len_arr_cpu,
|
||||
@@ -1052,6 +1067,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
|
||||
fast_verify_plan_kwargs: Optional[dict] = None,
|
||||
*,
|
||||
plan: KVLocPlan,
|
||||
qo_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_len_arr_cpu: Optional[torch.Tensor] = None,
|
||||
@@ -1068,11 +1084,11 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
dtype=torch.int32,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
self.attn_backend.kv_index_translator.fill_packed_read_stream(
|
||||
self.attn_backend.kv_index_translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=kv_indptr,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
)
|
||||
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
|
||||
@@ -1085,6 +1101,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
plan=plan,
|
||||
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
|
||||
)
|
||||
)
|
||||
@@ -1097,6 +1114,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
|
||||
paged_kernel_lens=paged_kernel_lens,
|
||||
paged_kernel_lens_sum=paged_kernel_lens_sum,
|
||||
translator=self.attn_backend.kv_index_translator,
|
||||
plan=plan,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1241,25 +1259,30 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
next_power_of_2(self.speculative_num_steps),
|
||||
next_power_of_2(bs),
|
||||
self.page_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -125,6 +126,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
or forward_mode.is_draft_extend_v2()
|
||||
):
|
||||
self._apply_decode_target_verify_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=forward_batch.batch_size,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
@@ -165,10 +167,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -200,10 +200,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -240,10 +238,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=seq_lens_k,
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=block_kv_indices
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -328,6 +324,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _apply_decode_target_verify_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -355,10 +352,9 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
if self.kv_index_translator.reads_are_translated:
|
||||
assert self.page_size == PAGE_SIZE
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=self.cuda_graph_kv_indices,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
# The plan's table reaches a verify's draft tail.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=self.cuda_graph_kv_indices[:bs]
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
|
||||
@@ -251,6 +251,7 @@ def _build_tbo_child_replay_fb_view(
|
||||
child_seq_lens_cpu = fb_view.seq_lens_cpu[seq_slice]
|
||||
parent_input_ids = getattr(fb_view, "input_ids", None)
|
||||
parent_out_cache_loc = getattr(fb_view, "out_cache_loc", None)
|
||||
plan = getattr(fb_view, "kv_loc_plan", None)
|
||||
return SimpleNamespace(
|
||||
batch_size=child_bs,
|
||||
forward_mode=fb_view.forward_mode,
|
||||
@@ -270,6 +271,15 @@ def _build_tbo_child_replay_fb_view(
|
||||
if parent_out_cache_loc is not None
|
||||
else None
|
||||
),
|
||||
# Unified memory refuses two-batch overlap, so the plan's reads stay in
|
||||
# `req_to_token`, which each child indexes at its own rows; each child
|
||||
# writes its own tokens of the plan's window.
|
||||
kv_loc_plan=plan,
|
||||
kv_loc_cols=(
|
||||
None
|
||||
if plan is None
|
||||
else plan.cols_slice(getattr(fb_view, "kv_loc_cols", None), tok_slice)
|
||||
),
|
||||
spec_info=child_spec_info,
|
||||
max_seq_len_override=fb_view.max_seq_len_override,
|
||||
)
|
||||
|
||||
@@ -404,7 +404,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
|
||||
if save_kv_cache:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
self._kv_write_loc(forward_batch),
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
k,
|
||||
k_rope,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.layers.dcp import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -68,6 +69,7 @@ if _is_cuda:
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
@@ -130,9 +132,6 @@ class ForwardMetadata:
|
||||
swa_attn_logits: Optional[torch.Tensor] = None
|
||||
# full->SWA translated out_cache_loc (SWA KV-store write target)
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None
|
||||
# PHYSICAL full-attn write target for the unified pool (eager: translated tensor;
|
||||
# cuda-graph: capture-stable buffer view). None for non-unified pools.
|
||||
out_cache_loc_full_physical: Optional[torch.Tensor] = None
|
||||
# Lean decode (persistent-grid partial-result buffers)
|
||||
lean_Mp: Optional[torch.Tensor] = None
|
||||
lean_Lp: Optional[torch.Tensor] = None
|
||||
@@ -503,6 +502,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _fill_kv_indptr_and_indices(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -512,17 +512,18 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if kv_indptr is None:
|
||||
kv_indptr = self.kv_indptr[: bs + 1]
|
||||
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
self.kv_index_translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
self.kv_index_translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens[:bs],
|
||||
indptr=kv_indptr,
|
||||
total_tokens=kv_indices.numel(),
|
||||
out=kv_indices,
|
||||
)
|
||||
return kv_indptr
|
||||
|
||||
def _update_decode_kv_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -551,7 +552,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
num_kv_splits_lens = dcp_seq_lens.clamp_min(1)
|
||||
else:
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
num_kv_splits_lens = seq_lens
|
||||
window_kv_indptr = self.window_kv_indptr
|
||||
@@ -560,6 +561,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_indptr, _, window_kv_lens, _ = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
seq_lens,
|
||||
@@ -577,6 +579,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_target_verify_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
spec_info,
|
||||
@@ -593,7 +596,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
window_kv_indptr = self.window_kv_indptr
|
||||
window_kv_indices = None
|
||||
@@ -607,6 +610,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
seq_lens[:bs],
|
||||
@@ -638,6 +642,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_dllm_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -654,12 +659,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
self._fill_kv_indptr_and_indices(
|
||||
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
if self.sliding_window_size is not None and self.sliding_window_size > 0:
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
prefix_lens,
|
||||
@@ -670,6 +676,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
def _update_draft_extend_buffers(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
@@ -705,7 +712,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
extend_seq_lens = torch.zeros(bs, dtype=torch.int32, device=seq_lens.device)
|
||||
kv_lens = torch.clamp(seq_lens - extend_seq_lens, min=0).to(torch.int32)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
return qo_indptr, kv_indptr, num_tokens_per_req
|
||||
|
||||
@@ -755,15 +762,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
return
|
||||
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
out_cache_loc_full_physical = self._fill_cuda_graph_write_locs(
|
||||
forward_batch, bs
|
||||
)
|
||||
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
@@ -772,25 +777,25 @@ class TritonAttnBackend(AttentionBackend):
|
||||
forward_mode,
|
||||
spec_info,
|
||||
swa_out_cache_loc,
|
||||
out_cache_loc_full_physical,
|
||||
)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
# Metadata view is reused from capture; just refill the buffers.
|
||||
self._fill_cuda_graph_write_locs(forward_batch, bs)
|
||||
# Metadata view is reused from capture; the write loc is the
|
||||
# runner's slot, so only the swa twin needs refilling.
|
||||
self._fill_cuda_graph_swa_out_cache_loc(forward_batch)
|
||||
|
||||
def _fill_cuda_graph_swa_out_cache_loc(
|
||||
self, forward_batch: ForwardBatch, in_capture: bool = False
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Refill the SWA write-target buffer from the batch's derived
|
||||
sliding-window write loc, returning the [:n] view (None for non-SWA /
|
||||
"""Refill the SWA write-target buffer with the batch's write ids in the
|
||||
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
|
||||
multi-step draft) so the captured store reads fresh slots on replay.
|
||||
"""
|
||||
if not self.use_sliding_window_kv_pool:
|
||||
@@ -807,24 +812,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
return self.cuda_graph_swa_out_cache_loc[:n]
|
||||
|
||||
def _fill_cuda_graph_write_locs(
|
||||
self, forward_batch: ForwardBatch, bs: int
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Runs BEFORE graph.replay(), so it reads the live post-compaction
|
||||
v2p; no-op for non-unified pools."""
|
||||
# The buffer exists only for a translating pool; return before naming it.
|
||||
if not self.kv_index_translator.is_translating:
|
||||
return None
|
||||
return self.kv_index_translator.fill_capture_write_loc(
|
||||
out=self.cuda_graph_out_cache_loc_full_physical,
|
||||
forward_batch=forward_batch,
|
||||
width=self.cuda_graph_out_cache_loc_full_physical.numel(),
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Init auxiliary variables for triton attention backend."""
|
||||
|
||||
@@ -862,6 +855,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
seq_lens_sum, dtype=torch.int64, device=self.device
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -875,6 +869,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.seq_lens,
|
||||
@@ -961,6 +956,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
seq_lens_sum, dtype=torch.int64, device=self.device
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -977,6 +973,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
) = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.seq_lens,
|
||||
@@ -1016,6 +1013,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
forward_batch.extend_prefix_lens,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -1030,6 +1028,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
) = update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
forward_batch.kv_loc_plan,
|
||||
forward_batch.req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
forward_batch.extend_prefix_lens,
|
||||
@@ -1054,8 +1053,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
swa_out_cache_loc = None
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -1074,11 +1073,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets,
|
||||
swa_attn_logits=swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=(
|
||||
forward_batch.out_cache_loc
|
||||
if self.kv_index_translator.is_translating
|
||||
else None
|
||||
),
|
||||
lean_Mp=lean_Mp,
|
||||
lean_Lp=lean_Lp,
|
||||
lean_Op=lean_Op,
|
||||
@@ -1201,22 +1195,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.kv_index_translator.is_translating:
|
||||
# Unified pool full-attention write-target buffer, refilled at replay
|
||||
# (-> KVWriteLoc.full_loc). Capture-stable, mirrors cuda_graph_swa_out_cache_loc.
|
||||
self.cuda_graph_out_cache_loc_full_physical = torch.zeros(
|
||||
(max_num_tokens,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _build_cuda_graph_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[SpecInput],
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None,
|
||||
out_cache_loc_full_physical: Optional[torch.Tensor] = None,
|
||||
) -> ForwardMetadata:
|
||||
"""Construct ForwardMetadata from the current cuda-graph buffer state.
|
||||
|
||||
@@ -1247,7 +1231,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets=None,
|
||||
swa_attn_logits=self.cuda_graph_swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
lean_Mp=self.cuda_graph_lean_Mp,
|
||||
lean_Lp=self.cuda_graph_lean_Lp,
|
||||
lean_Op=self.cuda_graph_lean_Op,
|
||||
@@ -1279,7 +1262,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
),
|
||||
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
return ForwardMetadata(
|
||||
@@ -1297,7 +1279,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
return ForwardMetadata(
|
||||
@@ -1323,13 +1304,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid forward mode: {forward_mode=} for CUDA Graph.")
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -1344,7 +1325,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if forward_mode.is_decode_or_idle():
|
||||
assert spec_info is None, "Multi-step cuda graph init is not done here."
|
||||
_, _, window_kv_lens, num_kv_splits_lens = self._update_decode_kv_buffers(
|
||||
bs, seq_lens, req_pool_indices
|
||||
plan, bs, seq_lens, req_pool_indices
|
||||
)
|
||||
self.get_num_kv_splits(
|
||||
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
|
||||
@@ -1356,13 +1337,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
elif forward_mode.is_target_verify():
|
||||
bs = len(req_pool_indices)
|
||||
self._update_target_verify_buffers(
|
||||
bs, seq_lens, spec_info, req_pool_indices
|
||||
plan, bs, seq_lens, spec_info, req_pool_indices
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
|
||||
self._update_dllm_buffers(plan, bs, seq_lens, req_pool_indices)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
self._update_draft_extend_buffers(
|
||||
bs, seq_lens, forward_mode, spec_info, req_pool_indices
|
||||
plan, bs, seq_lens, forward_mode, spec_info, req_pool_indices
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -1649,7 +1630,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
loc_info = KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
)
|
||||
if layer.k_scale is None:
|
||||
self._set_kv_buffer(forward_batch, layer, loc_info, k, v)
|
||||
@@ -2123,8 +2103,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
"window-layer extend before the metadata carried a "
|
||||
"sliding-window write loc"
|
||||
)
|
||||
elif self.forward_metadata.out_cache_loc_full_physical is not None:
|
||||
extend_kv_indices = self.forward_metadata.out_cache_loc_full_physical
|
||||
|
||||
# Capture batches may not have a spec_info, so use the attention
|
||||
# metadata's resolved uniform verify width when extend lengths are absent.
|
||||
@@ -2234,14 +2212,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
k.div_(layer.k_scale)
|
||||
self.token_to_kv_pool.set_kv_buffer(
|
||||
layer,
|
||||
# `full_loc` carries the pre-translated loc under the unified
|
||||
# pool, refreshed into a capture-stable buffer before replay —
|
||||
# translating inside set_kv_buffer would be captured and replay
|
||||
# a stale v2p. None (-> raw loc) for static pools.
|
||||
KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
),
|
||||
k,
|
||||
v,
|
||||
@@ -2253,7 +2226,6 @@ class TritonAttnBackend(AttentionBackend):
|
||||
KVWriteLoc.for_batch(
|
||||
forward_batch,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
full_loc=self.forward_metadata.out_cache_loc_full_physical,
|
||||
),
|
||||
k,
|
||||
v,
|
||||
@@ -2468,18 +2440,21 @@ class TritonMultiStepDraftBackend:
|
||||
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
|
||||
seq_lens_sum = num_seqs * self.max_context_len
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
forward_batch.req_pool_indices,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
src.row_stride,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
@@ -2488,7 +2463,9 @@ class TritonMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
|
||||
if call_fn is None:
|
||||
@@ -2590,6 +2567,7 @@ class TritonMultiStepDraftBackend:
|
||||
def update_sliding_window_buffer(
|
||||
window_kv_indptr,
|
||||
translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
sliding_window_size,
|
||||
seq_lens,
|
||||
@@ -2626,16 +2604,18 @@ def update_sliding_window_buffer(
|
||||
if isinstance(token_to_kv_pool, SWAKVPool)
|
||||
else None
|
||||
)
|
||||
translated = translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
translated = translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=window_kv_lens,
|
||||
indptr=window_kv_indptr,
|
||||
total_tokens=window_kv_indices.numel(),
|
||||
out=window_kv_indices,
|
||||
kv_start_idx=window_kv_start_idx,
|
||||
sliding_window=(
|
||||
translator.reads_are_translated
|
||||
kind=(
|
||||
IdSpaceKind.SLIDING_WINDOW
|
||||
if translator.reads_are_translated
|
||||
and isinstance(token_to_kv_pool, BaseSWAKVPool)
|
||||
else IdSpaceKind.FULL
|
||||
),
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
KVCacheAttentionAccessKind,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -1088,16 +1089,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
# fused kernel skips its page-table writes so the graph reads the
|
||||
# refreshed content through pointers baked at capture).
|
||||
metadata = self.forward_metadata
|
||||
# `cache_seqlens_int32` is what the attention kernels bound their
|
||||
# page-table reads by, and the fused metadata call above wrote it.
|
||||
# A target verify reads `draft_token_num` further than `seq_lens`
|
||||
# goes, so filling to `seq_lens` leaves those columns untranslated.
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.page_table,
|
||||
req_pool_indices=forward_batch.req_pool_indices[:bs],
|
||||
seq_lens=metadata.cache_seqlens_int32,
|
||||
sliding_window_out=metadata.swa_page_table,
|
||||
# The kernels bound their page-table reads by `cache_seqlens_int32`
|
||||
# (the fused metadata call above wrote it); the plan's table
|
||||
# reaches that far, a target verify's draft tail included.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan, out=metadata.page_table[:bs]
|
||||
)
|
||||
if metadata.swa_page_table is not None:
|
||||
self.kv_index_translator.copy_page_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
out=metadata.swa_page_table[:bs],
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
)
|
||||
# A capture batch carries no prepared write loc; zeros are the
|
||||
# page-0 sink.
|
||||
if (
|
||||
@@ -1110,8 +1113,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1153,7 +1156,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
metadata, req_pool_indices, metadata.cache_seqlens_int32
|
||||
)
|
||||
# The fused in-graph kernel also skips ragged batches, so refill the
|
||||
# SWA write-target buffer here (out_cache_loc -> SWA locs).
|
||||
# SWA write-target buffer here, from the plan.
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
n = forward_batch.out_cache_loc.shape[0]
|
||||
self.cuda_graph_swa_out_cache_loc[n:].zero_()
|
||||
@@ -1161,8 +1164,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1266,12 +1269,23 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
metadata.cu_seqlens_q = metadata.cu_seqlens_k
|
||||
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
kv_view = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
|
||||
)
|
||||
if kv_view.is_translated:
|
||||
# No fill kernel: the kernels take the tensor's own width/stride
|
||||
# and bound their reads by cache_seqlens.
|
||||
metadata.page_table = kv_view.ids
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
metadata.swa_page_table = (
|
||||
self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan,
|
||||
kind=IdSpaceKind.SLIDING_WINDOW,
|
||||
rows=forward_batch.batch_size,
|
||||
).ids
|
||||
if self.kv_index_translator.space(IdSpaceKind.SLIDING_WINDOW)
|
||||
is not None
|
||||
else None
|
||||
)
|
||||
else:
|
||||
has_swa = self._swa_kv_pool is not None
|
||||
metadata.page_table = torch.empty(
|
||||
@@ -1300,10 +1314,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
|
||||
# int64 scatter index (unlike the int32 read page table above).
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
metadata.swa_out_cache_loc = self.kv_index_translator.write_ids(
|
||||
forward_batch, IdSpaceKind.SLIDING_WINDOW
|
||||
)
|
||||
|
||||
self.forward_metadata = metadata
|
||||
|
||||
@@ -79,6 +79,7 @@ if is_flashinfer_available():
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -299,10 +300,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
# Tree-mask scratch is fetched from the target backend only.
|
||||
self.is_draft_runner = model_runner.is_draft_worker
|
||||
|
||||
# [:n] view of a capture-stable buffer on the cuda-graph path; None on
|
||||
# the eager path, which passes forward_batch.out_cache_loc through.
|
||||
self._decode_kernel_loc: Optional[torch.Tensor] = None
|
||||
self.cuda_graph_out_cache_loc_kernel: Optional[torch.Tensor] = None
|
||||
# Fused KV-scatter + q-concat on the decode dense-loc path (one launch
|
||||
# instead of set_mla_kv_buffer + concat_mla_absorb_q). Disabled under
|
||||
# async asserts: the fused path writes the pool directly and would
|
||||
@@ -411,6 +408,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _create_block_kv_indices(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
batch_size: int,
|
||||
max_blocks: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
@@ -430,11 +428,19 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
Returns:
|
||||
Block KV indices tensor
|
||||
"""
|
||||
dcp_enabled = get_parallel().dcp_enabled
|
||||
translated = self.kv_index_translator.is_translating and not dcp_enabled
|
||||
# A translated table holds the plan's page ids and reads the sink past
|
||||
# them: a draft extend pads every row's key length to its widest row,
|
||||
# and the padded queries (whose outputs are dropped) read that far.
|
||||
block_kv_indices = torch.full(
|
||||
(batch_size, max_blocks), -1, dtype=torch.int32, device=device
|
||||
(batch_size, max_blocks),
|
||||
0 if translated else -1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
if get_parallel().dcp_enabled:
|
||||
if dcp_enabled:
|
||||
self._fill_dcp_block_kv_indices(
|
||||
block_kv_indices,
|
||||
req_pool_indices,
|
||||
@@ -442,12 +448,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
)
|
||||
return block_kv_indices
|
||||
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=block_kv_indices,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
if translated:
|
||||
self.kv_index_translator.copy_page_table(plan, out=block_kv_indices)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
(
|
||||
@@ -480,13 +482,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
self.decode_cuda_graph_kv_indices = torch.full(
|
||||
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
# Unified pool: capture-stable buffer for the physical KV write loc, filled
|
||||
# out-of-graph in init_forward_metadata_out_graph so the in-graph
|
||||
# set_mla_kv_buffer captures no translate.
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.cuda_graph_out_cache_loc_kernel = torch.zeros(
|
||||
max_num_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
num_tokens_per_req = max_num_tokens // max_bs
|
||||
|
||||
if is_float4_e2m1fn_x2(self.data_type):
|
||||
@@ -603,6 +598,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
def _apply_cuda_graph_metadata(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -642,10 +638,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
# Update block indices for new sequences.
|
||||
if self.kv_index_translator.is_translating:
|
||||
self.kv_index_translator.fill_read_table(
|
||||
out=metadata.block_kv_indices,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
# The plan's table reaches a verify's draft tail.
|
||||
self.kv_index_translator.copy_page_table(
|
||||
plan, out=metadata.block_kv_indices[:bs]
|
||||
)
|
||||
else:
|
||||
create_flashmla_kv_indices_triton[
|
||||
@@ -755,6 +750,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
forward_batch.seq_lens.device,
|
||||
)
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
@@ -762,56 +758,13 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
)
|
||||
else:
|
||||
self._apply_cuda_graph_metadata(
|
||||
plan=forward_batch.kv_loc_plan,
|
||||
bs=bs,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
forward_mode=forward_mode,
|
||||
)
|
||||
|
||||
# Out-of-graph on capture AND every replay-prep, so the in-graph
|
||||
# set_mla_kv_buffer captures no translate.
|
||||
if self.kv_index_translator.is_translating and (
|
||||
forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()
|
||||
):
|
||||
# The captured kernel consumes the whole buffer, so the tail a
|
||||
# shorter replay leaves must go to slot 0 rather than live pages.
|
||||
self._decode_kernel_loc = self.kv_index_translator.fill_capture_write_loc(
|
||||
out=self.cuda_graph_out_cache_loc_kernel,
|
||||
forward_batch=forward_batch,
|
||||
width=self.cuda_graph_out_cache_loc_kernel.numel(),
|
||||
)
|
||||
else:
|
||||
self._decode_kernel_loc = None
|
||||
|
||||
def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc:
|
||||
"""The loc an unfused KV scatter must write at: the capture-stable
|
||||
buffer under a captured unified-pool decode, since the translate
|
||||
rebinds `out_cache_loc` to a fresh tensor the graph never recorded;
|
||||
the batch's own loc everywhere else.
|
||||
"""
|
||||
if self._decode_kernel_loc is not None:
|
||||
# Filled by `fill_capture_write_loc`, which translates it.
|
||||
return KVWriteLoc(self._decode_kernel_loc, physical=True)
|
||||
return KVWriteLoc.for_batch(forward_batch)
|
||||
|
||||
def _resolve_fused_write_loc(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Write loc for the fused fp8 KV scatter, or None when this batch is
|
||||
not covered by it.
|
||||
|
||||
Captured decode refills `_decode_kernel_loc` out of the graph, and the
|
||||
captured kernel must read that buffer. Eager decode on a unified pool
|
||||
has no such buffer, and the caller falls back to the unfused path.
|
||||
"""
|
||||
if self._decode_kernel_loc is not None:
|
||||
return self._decode_kernel_loc
|
||||
return (
|
||||
None
|
||||
if self.kv_index_translator.is_translating
|
||||
else forward_batch.out_cache_loc
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Initialize the metadata for a forward pass."""
|
||||
if self._kv_shard_pool is not None:
|
||||
@@ -821,7 +774,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
forward_batch,
|
||||
)
|
||||
|
||||
self._decode_kernel_loc = None
|
||||
# Delegate to parent for non-decode modes.
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
@@ -912,6 +864,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
max_seqlen_pad = self._calc_padded_blocks(max_seq)
|
||||
block_kv_indices = self._create_block_kv_indices(
|
||||
forward_batch.kv_loc_plan,
|
||||
bs,
|
||||
max_seqlen_pad,
|
||||
forward_batch.req_pool_indices,
|
||||
@@ -1287,7 +1240,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
# Fused: bf16->fp8 quantize + KV scatter + q concat
|
||||
# in one launch; None when not covered.
|
||||
@@ -1323,42 +1276,23 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert k is not None and k_rope is not None, (
|
||||
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
|
||||
)
|
||||
if self._decode_kernel_loc is not None:
|
||||
if merge_query and self._fused_set_kv_concat_q:
|
||||
# Fused: KV scatter + [q_nope | q_rope] concat in one
|
||||
# launch; None when the inputs are not covered.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=self._decode_kernel_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, self._kv_write_loc(forward_batch), k, k_rope
|
||||
)
|
||||
else:
|
||||
# eager (or static pool): out_cache_loc is physical.
|
||||
if (
|
||||
merge_query
|
||||
and self._fused_set_kv_concat_q
|
||||
and not self.kv_index_translator.is_translating
|
||||
):
|
||||
# Static pool only, conservatively.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=forward_batch.out_cache_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
# `out_cache_loc` is physical on every pool, eager or captured
|
||||
# (the runner's slot).
|
||||
if merge_query and self._fused_set_kv_concat_q:
|
||||
# Fused: KV scatter + [q_nope | q_rope] concat in one launch;
|
||||
# None when the inputs are not covered.
|
||||
query = self._set_kv_and_concat_q_fused(
|
||||
layer=layer,
|
||||
loc=forward_batch.out_cache_loc,
|
||||
k=k,
|
||||
k_rope=k_rope,
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
)
|
||||
if query is None:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
|
||||
# Prepare query tensor inline (already built when the fused save-KV
|
||||
# path ran)
|
||||
@@ -1515,7 +1449,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
|
||||
loc = self._resolve_fused_write_loc(forward_batch)
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is not None:
|
||||
# Fused: bf16->fp8 quantize + KV scatter + q concat
|
||||
# in one launch; None when not covered.
|
||||
@@ -1551,7 +1485,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
|
||||
)
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer, self._kv_write_loc(forward_batch), k, k_rope
|
||||
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
|
||||
)
|
||||
|
||||
# TODO refactor to avoid code duplication
|
||||
|
||||
@@ -604,6 +604,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
skip_attn_backend_init: Optional[bool] = None, # deprecated
|
||||
*,
|
||||
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
|
||||
return_kv_loc_plan: bool = False,
|
||||
) -> GenerationBatchResult:
|
||||
# Get forward batch from schedule batch
|
||||
if batch is not None:
|
||||
@@ -649,6 +650,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
expert_distribution_metrics=out.expert_distribution_metrics,
|
||||
routed_experts_output=out.routed_experts_output,
|
||||
indexer_topk_output=out.indexer_topk_output,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
|
||||
)
|
||||
|
||||
capture_pre_sample_logits(batch, forward_batch, logits_output)
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.utils.common import async_d2h as _async_d2h
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
|
||||
@@ -118,6 +119,11 @@ class GenerationBatchResult:
|
||||
# until the accepted path comes back over the relay.
|
||||
spec_verify_out_cache_loc: Optional[torch.Tensor] = None
|
||||
|
||||
# The plan the forward wrote its KV through, when the caller asked for it
|
||||
# (`return_kv_loc_plan`): a speculative worker's own writes into the same
|
||||
# slots take their ids from it.
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
|
||||
# PP+spec: [bs, spec_steps + 1] global node indices of the accepted path.
|
||||
# Every stage holds the KV for its own layers, so every stage has to compact
|
||||
# that path into its committed prefix; only the last stage can compute it.
|
||||
|
||||
@@ -510,10 +510,9 @@ class KVCacheConfigurator:
|
||||
token_to_kv_pool = None
|
||||
|
||||
# Unified-pool fast path: build req_to_token + token_to_kv pool + allocator
|
||||
# from one byte buffer, then return. Gated to the target worker by ROLE:
|
||||
# a compact-window DFLASH draft also passes req_to_token_pool=None, and
|
||||
# would allocate a SECOND unified byte buffer here. Supports hybrid
|
||||
# Mamba and hybrid SWA (not DSV4).
|
||||
# from one byte buffer, then return. Target worker only: a compact-window
|
||||
# DFLASH draft also passes req_to_token_pool=None. Supports hybrid Mamba
|
||||
# and hybrid SWA (not DSV4).
|
||||
if (
|
||||
get_memory().enable_unified_memory
|
||||
and req_to_token_pool is None
|
||||
@@ -581,7 +580,6 @@ class KVCacheConfigurator:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
placement = self._fused_draft_from_target_buffer(alloc)
|
||||
if placement is not None:
|
||||
# FUSED arm
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
draft_swa_layer_ids,
|
||||
)
|
||||
@@ -619,7 +617,6 @@ class KVCacheConfigurator:
|
||||
token_to_kv_pool_allocator=alloc,
|
||||
unified_memory_pool=None,
|
||||
)
|
||||
# PRIVATE arm
|
||||
draft_virtual_id_space = alloc.draft_virtual_id_space
|
||||
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
|
||||
"unified allocator virtual space smaller than the token "
|
||||
@@ -627,7 +624,7 @@ class KVCacheConfigurator:
|
||||
f"max_total_num_tokens={sizes.max_total_num_tokens}"
|
||||
)
|
||||
# Round UP to page alignment (paged draft backends view the
|
||||
# pool as (-1, page_size, H, D); the space is not aligned).
|
||||
# pool as (-1, page_size, H, D); the virtual space is not aligned).
|
||||
page = max(int(self.pool_page_size or 1), 1)
|
||||
draft_virtual_id_space = (
|
||||
(draft_virtual_id_space + page - 1) // page * page
|
||||
@@ -733,11 +730,8 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
def _fused_draft_from_target_buffer(self, alloc):
|
||||
"""The placement this draft binds to, or None for the private-pool
|
||||
fallback (the draft then owns a raw-virtual-indexed pool of its own).
|
||||
Target boot declines a placement whenever the fused region cannot
|
||||
serve the draft (`_fused_draft_decision`), so the private arm is the
|
||||
rollback lever, not a boot-order bug."""
|
||||
"""The placement this draft binds to, or None when the target placed
|
||||
none and the draft builds a private pool over the virtual id space."""
|
||||
if not (
|
||||
self.spec_algorithm.is_eagle() or self.spec_algorithm.is_dflash_family()
|
||||
):
|
||||
@@ -961,14 +955,10 @@ class KVCacheConfigurator:
|
||||
|
||||
def _fused_draft_decision(self):
|
||||
"""Whether, and where, the draft's layers fuse into the target's
|
||||
sub-pools. Fusion applies only for: unified memory ON, a unified target
|
||||
(hybrid-SWA or mamba hybrid, either full-pool kind), an EAGLE-family
|
||||
or DFLASH/DSPARK algorithm whose draft config was loaded at target
|
||||
boot. It declines, keeping the draft's private pool, for a non-MHA
|
||||
draft, a draft backend off the translated MHA rails, DCP, HiCache,
|
||||
host-pool decode retraction, or an explicit draft KV dtype unlike the
|
||||
host's; `place_fused_draft` then admits or declines the draft's layer
|
||||
kinds."""
|
||||
sub-pools. An empty decision means fusion does not apply (unified
|
||||
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
|
||||
loaded at target boot); a declined one says why the draft cannot fuse.
|
||||
The pool factories decide whether a decline refuses the boot."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
draft_kv_profile,
|
||||
@@ -998,10 +988,9 @@ class KVCacheConfigurator:
|
||||
"the fused region holds dense MHA K/V rows"
|
||||
)
|
||||
)
|
||||
# The backend(s) the draft runner will run. A DFLASH-family draft
|
||||
# worker resolves one as `build_draft_tp_worker` does; an EAGLE draft
|
||||
# runs the published draft backend, else the target's prefill/decode
|
||||
# pair. A published draft backend carries model-hook declarations.
|
||||
# The draft runner's backend, resolved as its worker will: an EAGLE
|
||||
# draft with no backend of its own runs the target's prefill/decode
|
||||
# pair. The published draft backend carries model-hook declarations.
|
||||
if self.spec_algorithm.is_dflash_family():
|
||||
from sglang.srt.speculative.draft_worker_common import (
|
||||
resolve_draft_worker_attention_backend,
|
||||
@@ -1066,10 +1055,9 @@ class KVCacheConfigurator:
|
||||
else 1
|
||||
)
|
||||
# A fused draft stores its rows in the host's KV dtype. An explicit
|
||||
# draft dtype resolves without the draft model, as the draft runner
|
||||
# resolves it, so a mismatch is declined here; `auto` follows the
|
||||
# draft's quant config, which only the draft runner knows, and is
|
||||
# checked when it binds.
|
||||
# draft dtype resolves here as the draft runner will; `auto` follows
|
||||
# the draft's quant config, which only the draft runner knows, so it
|
||||
# is checked when the draft binds.
|
||||
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
|
||||
if draft_kv_cache_dtype not in (None, "auto"):
|
||||
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
|
||||
@@ -1098,11 +1086,10 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
|
||||
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
|
||||
pad) for the boot solve's cell model. Single source of truth: assembled
|
||||
through the same spec the pool factory builds, so the priced entry and
|
||||
the allocated entry cannot drift. None when the draft does not fuse
|
||||
into that sub-pool."""
|
||||
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
|
||||
pad), built from the same spec as the pool factory's so the priced and
|
||||
allocated entries agree; None when the draft does not fuse into that
|
||||
sub-pool."""
|
||||
placement = self._fused_draft_decision().placement
|
||||
if placement is None or sub_pool_name != "full":
|
||||
return None
|
||||
@@ -1150,14 +1137,11 @@ class KVCacheConfigurator:
|
||||
|
||||
def _fused_draft_for_mamba_factory(self):
|
||||
"""`_fused_draft_for_pool_factory` for a mamba-hybrid host. Its unified
|
||||
buffer takes the WHOLE profiled KV budget (`unified_total_bytes`), so a
|
||||
private draft pool -- sized to the full virtual id span -- would be
|
||||
allocated on top of it, unbudgeted. Until the mamba solve prices that
|
||||
pool, an EAGLE-family or DFLASH draft that does not fuse is refused here
|
||||
instead of overcommitting GPU memory. A DSPARK draft that does not fuse
|
||||
keeps its private pool, unpriced: refusing it would refuse DSPARK on
|
||||
these hosts whenever its draft declines fusion, as the Kimi-Linear
|
||||
default (a trtllm_mha draft) does."""
|
||||
buffer takes the whole profiled KV budget, so a private draft pool
|
||||
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
|
||||
does not fuse is refused. A DSPARK draft that does not fuse keeps its
|
||||
private pool, unpriced, so DSPARK still boots where its draft declines
|
||||
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
|
||||
decision = self._fused_draft_decision()
|
||||
placement = self._fused_draft_for_pool_factory(decision)
|
||||
if (
|
||||
@@ -1169,18 +1153,16 @@ class KVCacheConfigurator:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory + EAGLE/EAGLE3/DFLASH on a mamba-hybrid "
|
||||
"target needs the draft's KV fused into the target's pages, but "
|
||||
f"this draft would keep a private pool ({reason}). A mamba host's "
|
||||
"unified buffer takes the whole KV budget, so a private draft "
|
||||
"pool would overcommit GPU memory."
|
||||
f"this draft cannot fuse ({reason}). A private draft pool would "
|
||||
"overcommit the KV budget the unified buffer already takes."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _fused_draft_for_swa_factory(self):
|
||||
"""`_fused_draft_for_pool_factory` for a hybrid-SWA host. Its boot solve
|
||||
prices a private EAGLE draft pool at the target's per-token size, not
|
||||
at the draft's own heads, head_dim and KV dtype, so a draft that does
|
||||
not fuse is refused here, as on a mamba host, instead of overcommitting
|
||||
GPU memory."""
|
||||
the draft's own geometry and KV dtype, so an EAGLE draft that does not
|
||||
fuse is refused rather than overcommit GPU memory."""
|
||||
decision = self._fused_draft_decision()
|
||||
placement = self._fused_draft_for_pool_factory(decision)
|
||||
if (
|
||||
@@ -1192,15 +1174,14 @@ class KVCacheConfigurator:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 on a hybrid-SWA target "
|
||||
"needs the draft's KV fused into the target's pages, but this "
|
||||
f"draft would keep a private pool ({reason}), which the unified "
|
||||
"pool's sizing does not price."
|
||||
f"draft cannot fuse ({reason}), and the unified pool's sizing "
|
||||
"does not price a private draft pool."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _fused_draft_for_pool_factory(self, decision):
|
||||
"""The placement a pool factory is handed, from ONE resolved decision,
|
||||
so the boot log reports exactly that placement: a declined fusion and
|
||||
an engaged one otherwise look identical from outside."""
|
||||
"""The placement a pool factory is handed. It logs the decision it
|
||||
returns, so the boot log shows whether fusion engaged or why not."""
|
||||
if decision.placement is None:
|
||||
if decision.declined is not None:
|
||||
logger.warning("fused draft KV disabled: %s", decision.declined)
|
||||
|
||||
@@ -23,18 +23,29 @@ A KV slot can be named in two id spaces:
|
||||
|
||||
The two coincide on a plain pool, so nothing here does any work there.
|
||||
|
||||
Backends get a `KVIndexTable`, which answers "what do I gather from, and
|
||||
which row is mine?":
|
||||
A pool can hold several sub-pools (full attention, sliding window), each with
|
||||
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
|
||||
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
|
||||
once, by the readers below and nowhere else; a backend never translates. The
|
||||
plan's read table exists only once a table reader asks for it (`read_table`,
|
||||
`copy_page_table`), built into that reader's own capture-stable table when it
|
||||
is the first; later readers take it, earlier ones translate in their own
|
||||
gather. The readers answer "what do I gather from, and which row is mine?"
|
||||
with a `KVIndexTable`:
|
||||
|
||||
ids[row_ids[b], pos]
|
||||
|
||||
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
|
||||
objects - no copy, no kernel)
|
||||
unified : ids = a built array of physical ids,
|
||||
row_ids = arange(batch_size)
|
||||
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
|
||||
objects - no copy, no kernel)
|
||||
shared table : ids = the plan's array of physical page ids,
|
||||
row_ids = arange(batch_size)
|
||||
no table : ids = req_to_token, row_ids = req_pool_indices, and
|
||||
v2p = the sub-pool's page table, applied by the gather
|
||||
|
||||
Backends call their own copy a *page table* (fa3) or a *block table*
|
||||
(trtllm); here it is the **index table**.
|
||||
(trtllm); here it is the **index table**. `read_source` serves a gather
|
||||
kernel, `pack_read_stream` a CSR stream, and `read_table` / `copy_page_table`
|
||||
a page-table kernel.
|
||||
|
||||
Converting only ever rewrites the page number and keeps the in-page offset, so
|
||||
one page-granular table serves both kinds of consumer: a block-table backend
|
||||
@@ -42,16 +53,15 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
|
||||
|
||||
token_id = entry * entry_page_size + pos % entry_page_size
|
||||
|
||||
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
|
||||
ids at ForwardBatch construction - the earliest consumer can snapshot it
|
||||
right after. The sliding-window write loc is derived at the same moment as
|
||||
read table, into the same index table.
|
||||
The one translation outside a plan is DCP's read production
|
||||
(`translate_dcp_read_ids`): which ids a rank reads is only known where the
|
||||
DCP index kernels select them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from typing import Optional, Tuple
|
||||
import functools
|
||||
from typing import Dict, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -63,6 +73,7 @@ from sglang.kernels.ops.kvcache.kv_read_table import (
|
||||
build_kv_read_table,
|
||||
build_kv_read_table_packed,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
@@ -71,30 +82,36 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
|
||||
)
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.kv_loc_plan import (
|
||||
IdSpace,
|
||||
IdSpaceKind,
|
||||
KVLocPlan,
|
||||
pad_with_sink,
|
||||
window_read_extent,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
|
||||
class KVReadTables(msgspec.Struct, frozen=True):
|
||||
"""One capture-stable destination, however many id spaces the pool has.
|
||||
|
||||
A backend holds this and hands it back to `build_index_table(into=...)`;
|
||||
it never has to know whether there is a sliding-window space behind it.
|
||||
"""
|
||||
|
||||
full: torch.Tensor
|
||||
sliding_window: Optional[torch.Tensor]
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
class KVIndexTable(msgspec.Struct, frozen=True):
|
||||
"""Collection of what one batch gathers from."""
|
||||
"""What one batch gathers from, and how its entries become the ids a
|
||||
kernel reads: lane ``b``'s entry ``j`` is ``ids[row_ids[b] * row_stride +
|
||||
j // entry_page_size]``, expanded to tokens when an entry is a page, then
|
||||
mapped through ``v2p`` (page-granular, at the translator's page size) when
|
||||
one is given. A gather kernel executes exactly this; it never decides
|
||||
whether or how to translate."""
|
||||
|
||||
ids: torch.Tensor # 2-D array of KV ids to gather from
|
||||
row_ids: torch.Tensor # which row belongs to batch lane b
|
||||
row_stride: int # stride between rows of `ids`, in elements
|
||||
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
|
||||
is_translated: bool # entries are already physical ids
|
||||
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
|
||||
is_translated: bool # the ids a gather yields are physical
|
||||
# Virtual rows a gather translates on the way out (no shared table this
|
||||
# iteration); None when the entries are already what the kernel reads.
|
||||
v2p: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class KVIndexTranslator:
|
||||
@@ -108,6 +125,7 @@ class KVIndexTranslator:
|
||||
token_to_kv_pool,
|
||||
page_size: int,
|
||||
device: str,
|
||||
is_draft_worker: bool = False,
|
||||
):
|
||||
self.req_to_token = req_to_token
|
||||
self.page_size = page_size
|
||||
@@ -125,55 +143,104 @@ class KVIndexTranslator:
|
||||
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
|
||||
)
|
||||
self.is_translating = is_unified_target or is_fused_draft
|
||||
self._spaces: Dict[IdSpaceKind, IdSpace] = {}
|
||||
if self.is_translating:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
self._capture_page_size = alloc.page_size
|
||||
self._full_v2p_table = alloc.full_v2p_page_table
|
||||
self._full_p2v_table = alloc.full_p2v_page_table
|
||||
self._translate_full = alloc.translate_kv_loc
|
||||
# The WRITE loc is the one id that arrives DCP-WIDENED: read indices
|
||||
# are collapsed by the DCP index kernels, `out_cache_loc` still
|
||||
# carries the owner rule in `loc % dcp_size`. Identity with the read
|
||||
# translate when dcp_size == 1.
|
||||
self._translate_write_full = alloc.translate_write_loc
|
||||
# DCP read ids stay WIDENED to the consumer: selecting this rank's
|
||||
# share changes the length, so only the production site can do it.
|
||||
self.defer_read_translate = get_parallel().attn_dcp_size > 1
|
||||
# Keyed by the allocator, which owns the tables: a fused draft's
|
||||
# runner shares the target's full sub-pool, and so its key.
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(
|
||||
key=(IdSpaceKind.FULL, id(alloc)),
|
||||
# The WRITE loc is the one id that arrives DCP-WIDENED: read
|
||||
# indices are collapsed by the DCP index kernels, `out_cache_loc`
|
||||
# still carries the owner rule in `loc % dcp_size`.
|
||||
write=alloc.translate_write_loc,
|
||||
read_v2p=None if self.defer_read_translate else self._full_v2p_table,
|
||||
)
|
||||
routes_window_layers = is_unified_target or isinstance(
|
||||
token_to_kv_pool, BaseSWAKVPool
|
||||
)
|
||||
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
|
||||
self._swa_v2p_table = alloc.swa_v2p_page_table
|
||||
self._swa_write_loc_from_full = self._swa_write_loc_unified
|
||||
else:
|
||||
self._swa_v2p_table = None
|
||||
self._swa_write_loc_from_full = None
|
||||
# Straight from the virtual ids through the sliding-window
|
||||
# sub-pool's own table.
|
||||
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
|
||||
key=(IdSpaceKind.SLIDING_WINDOW, id(alloc)),
|
||||
write=alloc.translate_loc_from_full_to_swa,
|
||||
read_v2p=(
|
||||
None if self.defer_read_translate else alloc.swa_v2p_page_table
|
||||
),
|
||||
)
|
||||
else:
|
||||
self._capture_page_size = page_size
|
||||
self._full_v2p_table = None
|
||||
self._full_p2v_table = None
|
||||
self._translate_full = None
|
||||
self._translate_write_full = None
|
||||
self.defer_read_translate = False
|
||||
self._swa_v2p_table = None
|
||||
parallel = get_parallel()
|
||||
if _is_npu and parallel.dcp_enabled and not is_draft_worker:
|
||||
# An NPU DCP target writes its rank's share of the window in
|
||||
# rank-local slots (-1: another rank's); the scheduler,
|
||||
# `req_to_token` and the replicated draft pool keep the
|
||||
# allocator-global ones the window is planned in.
|
||||
localize = functools.partial(
|
||||
localize_dcp_indices,
|
||||
dcp_size=parallel.dcp_size,
|
||||
dcp_rank=parallel.dcp_rank,
|
||||
interleave_size=page_size,
|
||||
)
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(
|
||||
key=(IdSpaceKind.FULL, "dcp-rank-local"), write=localize
|
||||
)
|
||||
else:
|
||||
localize = None
|
||||
self._spaces[IdSpaceKind.FULL] = IdSpace(key=(IdSpaceKind.FULL, None))
|
||||
# `translate_loc_from_full_to_swa` is abstract on `BaseSWAKVPool`,
|
||||
# which is also what the backends' `_resolve_swa_kv_pool` keys on.
|
||||
self._swa_write_loc_from_full = (
|
||||
token_to_kv_pool.translate_loc_from_full_to_swa
|
||||
if isinstance(token_to_kv_pool, BaseSWAKVPool)
|
||||
and (
|
||||
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
or token_to_kv_pool.request_window is None
|
||||
# Reads stay in `req_to_token`; the backends map them themselves.
|
||||
if isinstance(token_to_kv_pool, BaseSWAKVPool) and (
|
||||
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
or token_to_kv_pool.request_window is None
|
||||
):
|
||||
to_swa = token_to_kv_pool.translate_loc_from_full_to_swa
|
||||
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
|
||||
key=(IdSpaceKind.SLIDING_WINDOW, id(token_to_kv_pool)),
|
||||
write=(
|
||||
to_swa
|
||||
if localize is None
|
||||
else lambda ids: to_swa(localize(ids))
|
||||
),
|
||||
)
|
||||
else None
|
||||
)
|
||||
|
||||
self._rows: Optional[torch.Tensor] = (
|
||||
torch.arange(req_to_token.shape[0], dtype=torch.int64, device=device)
|
||||
if self.is_translating
|
||||
else None
|
||||
)
|
||||
self._index_table_memo: Optional[Tuple[weakref.ref, KVIndexTable]] = None
|
||||
|
||||
def space(self, kind: IdSpaceKind) -> Optional[IdSpace]:
|
||||
"""How this runner's ``kind`` sub-pool names the iteration's slots, or
|
||||
None when its pool has no such sub-pool."""
|
||||
return self._spaces.get(kind)
|
||||
|
||||
def write_ids(
|
||||
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""The ids ``forward_batch`` writes in this runner's ``kind`` sub-pool,
|
||||
taken from its plan and padded with the sink to its `out_cache_loc`'s
|
||||
length (a padded batch, a captured graph's buffer). None when there is
|
||||
no write loc or no such sub-pool."""
|
||||
plan = getattr(forward_batch, "kv_loc_plan", None)
|
||||
out_cache_loc = getattr(forward_batch, "out_cache_loc", None)
|
||||
if plan is None or out_cache_loc is None:
|
||||
return None
|
||||
ids = plan.write_ids(
|
||||
self, kind=kind, cols=getattr(forward_batch, "kv_loc_cols", None)
|
||||
)
|
||||
return pad_with_sink(ids, out_cache_loc.shape[0])
|
||||
|
||||
def capture_token_capacity(self, max_token_pool_size: int) -> int:
|
||||
"""Host capture rows are indexed by request-token IDs, not kernel IDs.
|
||||
@@ -185,240 +252,305 @@ class KVIndexTranslator:
|
||||
return self._full_v2p_table.numel() * self._capture_page_size
|
||||
return max_token_pool_size + self.page_size
|
||||
|
||||
def full_flat_v2p(self) -> Optional[torch.Tensor]:
|
||||
"""The full-side v2p page table for a kernel that translates flat ids
|
||||
itself, or ``None`` when this runner does not translate (pass-through
|
||||
and static pools)."""
|
||||
if not self.is_translating:
|
||||
return None
|
||||
return self._full_v2p_table
|
||||
# -- the iteration's plan --------------------------------------------------
|
||||
|
||||
# -- per-batch view --------------------------------------------------------
|
||||
def plan(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
) -> KVLocPlan:
|
||||
"""This iteration's ids, translated once. ``write_virtual`` is the
|
||||
iteration's write window (``[batch, window]`` flattened when its
|
||||
forwards write columns of it); ``read_extent`` is how far past
|
||||
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
|
||||
return KVLocPlan(
|
||||
source=self,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=write_virtual,
|
||||
read_extent=read_extent,
|
||||
write_slots=write_slots,
|
||||
)
|
||||
|
||||
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
|
||||
"""The plan of a forward that is its own iteration, from its own
|
||||
fields: its ``out_cache_loc`` is the write window (virtual, or with
|
||||
``runner_slots`` a runner's own buffer), and its reads reach
|
||||
`window_read_extent` past its lengths."""
|
||||
write_ids = forward_batch.out_cache_loc
|
||||
seq_lens = getattr(forward_batch, "seq_lens", None)
|
||||
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
|
||||
encoder_lens = getattr(forward_batch, "encoder_lens", None)
|
||||
if (
|
||||
encoder_lens is not None
|
||||
and seq_lens is not None
|
||||
and self.reads_are_translated
|
||||
):
|
||||
# An encoder-decoder row holds its encoder tokens ahead of the
|
||||
# decoder's, and its self-attention reads past `seq_lens`.
|
||||
seq_lens = seq_lens + encoder_lens
|
||||
encoder_lens_cpu = getattr(forward_batch, "encoder_lens_cpu", None)
|
||||
seq_lens_cpu = (
|
||||
None
|
||||
if seq_lens_cpu is None or encoder_lens_cpu is None
|
||||
else seq_lens_cpu + torch.tensor(encoder_lens_cpu).to(seq_lens_cpu)
|
||||
)
|
||||
return self.plan(
|
||||
req_pool_indices=getattr(forward_batch, "req_pool_indices", None),
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=None if runner_slots else write_ids,
|
||||
write_slots=write_ids if runner_slots else None,
|
||||
read_extent=window_read_extent(
|
||||
getattr(forward_batch, "forward_mode", None),
|
||||
getattr(forward_batch, "spec_info", None),
|
||||
write_ids,
|
||||
getattr(forward_batch, "batch_size", 0),
|
||||
),
|
||||
)
|
||||
|
||||
def bind_runner_slots(self, forward_batch) -> None:
|
||||
"""Bind a batch a runner builds over its own buffers, to capture a
|
||||
graph or to warm up with. Its write ids are the runner's buffer, used
|
||||
as it is; its reads plan over the runner's own rows and lengths."""
|
||||
self.own_plan(forward_batch, runner_slots=True).bind(forward_batch, self)
|
||||
|
||||
def build_iteration_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
space: IdSpace,
|
||||
*,
|
||||
rows: Optional[int],
|
||||
previous: Optional[KVIndexTable],
|
||||
into: Optional[torch.Tensor] = None,
|
||||
) -> KVIndexTable:
|
||||
"""Build ``plan``'s page table in ``space`` for `KVLocPlan.read_table`:
|
||||
the passthrough where reads stay virtual, else one build over ``[0,
|
||||
seq_lens + read_extent)``. Rows past the plan's batch read the sink; a
|
||||
request wider than ``previous`` copies its rows rather than translating
|
||||
again. ``into``, the asking reader's capture-stable table, is built in
|
||||
place when wide enough and then serves as the plan's table."""
|
||||
if space.read_v2p is None:
|
||||
return self._passthrough_table(plan.req_pool_indices)
|
||||
bs = int(plan.req_pool_indices.numel())
|
||||
rows = max(bs, rows or 0)
|
||||
if previous is not None:
|
||||
out = torch.zeros(
|
||||
(rows, previous.ids.shape[1]), dtype=torch.int32, device=self.device
|
||||
)
|
||||
out[: previous.ids.shape[0]].copy_(previous.ids)
|
||||
else:
|
||||
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
|
||||
slc = plan.seq_lens_cpu
|
||||
if slc is not None and slc.numel() > 0:
|
||||
max_seq = int(slc.max()) + plan.read_extent
|
||||
width = min(max(-(-max_seq // self.page_size), 1), row_pages)
|
||||
elif into is not None:
|
||||
# No host lengths: a row's speculative headroom outruns the
|
||||
# captured table, which is as wide as its graph reads.
|
||||
width = min(into.shape[1], row_pages)
|
||||
else:
|
||||
width = row_pages
|
||||
# `zero_tail` writes the sink past each live prefix, so neither a
|
||||
# reused `into` nor an `empty` table keeps stale columns.
|
||||
if into is not None and into.shape[0] >= rows and into.shape[1] >= width:
|
||||
out = into[:rows, :width]
|
||||
else:
|
||||
out = torch.empty((rows, width), dtype=torch.int32, device=self.device)
|
||||
if rows > bs:
|
||||
out[bs:].zero_()
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=plan.req_pool_indices,
|
||||
seq_lens=plan.seq_lens,
|
||||
v2p=space.read_v2p,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out,
|
||||
zero_tail=True,
|
||||
seq_len_delta=plan.read_extent,
|
||||
)
|
||||
return KVIndexTable(
|
||||
ids=out,
|
||||
row_ids=self._rows[:rows],
|
||||
row_stride=out.stride(0),
|
||||
entry_page_size=self.page_size,
|
||||
is_translated=True,
|
||||
)
|
||||
|
||||
# -- readers ---------------------------------------------------------------
|
||||
|
||||
def _reads_translated(self, kind: IdSpaceKind) -> bool:
|
||||
space = self._spaces.get(kind)
|
||||
return space is not None and space.read_v2p is not None
|
||||
|
||||
def read_source(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
bs: int,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
) -> KVIndexTable:
|
||||
"""The row source a gather kernel reads lane ``b``'s ``kind`` ids from:
|
||||
the plan's table once a table reader has had it built, else the virtual
|
||||
`req_to_token` rows with ``v2p`` for the kernel's gather to apply (no
|
||||
table is built for it). The plain `req_to_token` passthrough where
|
||||
reads stay virtual (a static pool, or DCP)."""
|
||||
if not self._reads_translated(kind):
|
||||
return self._passthrough_table(req_pool_indices)
|
||||
if plan.has_read_table(kind):
|
||||
return self._plan_table(plan, kind=kind, rows=bs)
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=True,
|
||||
v2p=self.space(kind).read_v2p,
|
||||
)
|
||||
|
||||
def read_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
rows: Optional[int] = None,
|
||||
) -> KVIndexTable:
|
||||
"""The plan's ``kind`` page table, exactly ``rows`` rows when given
|
||||
(lanes past the plan's batch read the sink); the ``req_to_token``
|
||||
passthrough where reads stay virtual (a static or private pool, or DCP,
|
||||
where the producing kernel selects this rank's share)."""
|
||||
if self._reads_translated(kind):
|
||||
return self._plan_table(plan, kind=kind, rows=rows)
|
||||
return self._passthrough_table(plan.req_pool_indices)
|
||||
|
||||
def _plan_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
kind: IdSpaceKind,
|
||||
rows: Optional[int] = None,
|
||||
into: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
)
|
||||
return plan.read_table(kind=kind, rows=rows, into=into)
|
||||
|
||||
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=False,
|
||||
)
|
||||
|
||||
def pack_read_stream(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
|
||||
over; return whether they are physical. Packed from the plan's table
|
||||
once a table reader has had it built, else gathered and translated
|
||||
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
|
||||
(a static pool, or DCP) for the caller to finish; a static SWA pool can
|
||||
fuse its full->swa table into the gather as ``token_mapping``."""
|
||||
bs = int(seq_lens.numel())
|
||||
if self._reads_translated(kind) and not plan.has_read_table(kind):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
)
|
||||
# `seq_lens` sizes the batch: a caller may hold a wider
|
||||
# req_pool_indices (a padded graph buffer), whose extra lanes have
|
||||
# no length to bound.
|
||||
assert req_pool_indices.numel() >= bs, (
|
||||
f"pack_read_stream: {bs} stream lanes, "
|
||||
f"{req_pool_indices.numel()} request rows"
|
||||
)
|
||||
build_kv_read_table_packed(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=seq_lens,
|
||||
v2p=self.space(kind).read_v2p,
|
||||
indptr=indptr,
|
||||
page_size=self.page_size,
|
||||
max_tokens=out.numel(),
|
||||
out=out,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
return True
|
||||
# The plan's table or a passthrough: the entries are the stream's ids.
|
||||
src = self.read_source(
|
||||
plan, req_pool_indices=req_pool_indices, bs=bs, kind=kind
|
||||
)
|
||||
assert src.v2p is None, "pack_read_stream: a table-less translating read"
|
||||
assert token_mapping is None or not src.is_translated
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
src.ids,
|
||||
src.row_ids[:bs],
|
||||
seq_lens,
|
||||
indptr,
|
||||
kv_start_idx,
|
||||
out,
|
||||
src.row_stride,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return src.is_translated or token_mapping is not None
|
||||
|
||||
def copy_page_table(
|
||||
self,
|
||||
plan: KVLocPlan,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
) -> None:
|
||||
"""Fill ``out``, a captured graph's page table (padded lanes read the
|
||||
sink), with the plan's ``kind`` table. The plan's first table reader
|
||||
builds straight into ``out``, which then serves as the plan's table; a
|
||||
later one copies it. Columns past the plan's width keep stale values,
|
||||
which kernels never read past their own lengths."""
|
||||
assert self._reads_translated(kind), (
|
||||
"copy_page_table: reads stay virtual here (a non-unified pool, or "
|
||||
"DCP, where the caller selects this rank's share itself)"
|
||||
)
|
||||
rows = out.shape[0]
|
||||
first = not plan.has_read_table(kind)
|
||||
table = self._plan_table(
|
||||
plan, kind=kind, rows=rows, into=out if first else None
|
||||
)
|
||||
if table.ids.data_ptr() == out.data_ptr():
|
||||
return # built in place, or this very buffer is the plan's table
|
||||
width = min(out.shape[1], table.ids.shape[1])
|
||||
out[:, :width].copy_(table.ids[:rows, :width])
|
||||
|
||||
# -- dispositions ----------------------------------------------------------
|
||||
|
||||
@property
|
||||
def reads_are_translated(self) -> bool:
|
||||
"""Whether a read this translator fills comes out physical. False
|
||||
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
|
||||
``translate_dcp_read_ids`` to finish."""
|
||||
return self.is_translating and not self.defer_read_translate
|
||||
|
||||
def fill_packed_read_stream(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
total_tokens: int,
|
||||
out: torch.Tensor,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
|
||||
report whether they came out translated.
|
||||
|
||||
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
|
||||
Unified: one fused gather-and-translate, so no caller needs a
|
||||
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
|
||||
resident token, a length the pool bounds.
|
||||
|
||||
``sliding_window`` selects the swa sub-pool's own id space, built from
|
||||
VIRTUAL ids and never chained through full-physical. A ``False`` return
|
||||
means the ids are still VIRTUAL: the DCP path defers translation to
|
||||
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
|
||||
table as ``token_mapping`` to fuse that translation into the gather.
|
||||
"""
|
||||
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
|
||||
# (the padded graph buffer), and the extra lanes have no length to bound.
|
||||
bs = int(seq_lens.numel())
|
||||
assert req_pool_indices.numel() >= bs, (
|
||||
f"fill_packed_read_stream: {req_pool_indices.numel()} req rows for "
|
||||
f"{bs} lengths"
|
||||
)
|
||||
req_pool_indices = req_pool_indices[:bs]
|
||||
|
||||
if not self.reads_are_translated:
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
self.req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
indptr,
|
||||
kv_start_idx,
|
||||
out,
|
||||
self.req_to_token.stride(0),
|
||||
ENTRY_PAGE_SIZE=1,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return token_mapping is not None
|
||||
|
||||
if sliding_window:
|
||||
assert self._swa_v2p_table is not None, (
|
||||
"fill_packed_read_stream: sliding_window on a pool with no swa sub-pool"
|
||||
)
|
||||
build_kv_read_table_packed(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._swa_v2p_table if sliding_window else self._full_v2p_table,
|
||||
indptr=indptr,
|
||||
page_size=self.page_size,
|
||||
max_tokens=total_tokens,
|
||||
out=out,
|
||||
kv_start_idx=kv_start_idx,
|
||||
)
|
||||
return True
|
||||
|
||||
def build_index_table(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
max_pages: Optional[int] = None,
|
||||
into: Optional[KVReadTables] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> KVIndexTable:
|
||||
"""The one per-batch entry point.
|
||||
|
||||
Non-unified: the raw ``(req_to_token, req_pool_indices)`` passthrough,
|
||||
no tensor ops and no copies. Unified: fills each row's live prefix and
|
||||
returns the table WHOLE, so a caller needing a stable pointer (a
|
||||
captured graph bakes it) passes its own tables in ``into``;
|
||||
``into=None`` allocates of width ``max_pages`` instead.
|
||||
|
||||
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
|
||||
verify contract (draft KV read back from the pool). An eager caller's
|
||||
``max_pages`` must already cover the delta.
|
||||
"""
|
||||
if not self.is_translating or self.defer_read_translate:
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
row_ids=req_pool_indices,
|
||||
row_stride=self.req_to_token.stride(0),
|
||||
entry_page_size=1,
|
||||
is_translated=False,
|
||||
sliding_window_ids=None,
|
||||
)
|
||||
|
||||
bs = int(req_pool_indices.numel())
|
||||
if into is not None:
|
||||
out_full = into.full
|
||||
out_swa = into.sliding_window
|
||||
# A caller-owned table may be padded wider than req_to_token's span
|
||||
# (trtllm_mla / flashmla pad to a page-count bound); the columns
|
||||
# past it have no source to read, so stop there.
|
||||
width = min(
|
||||
out_full.shape[1] if max_pages is None else max_pages,
|
||||
-(-self.req_to_token.shape[1] // self.page_size),
|
||||
)
|
||||
else:
|
||||
assert max_pages is not None, (
|
||||
"KVIndexTranslator.build_index_table: allocating needs max_pages "
|
||||
"(from the batch's seq_lens_cpu max)"
|
||||
)
|
||||
width = max_pages
|
||||
out_full = torch.zeros((bs, width), dtype=torch.int32, device=self.device)
|
||||
out_swa = (
|
||||
torch.zeros((bs, width), dtype=torch.int32, device=self.device)
|
||||
if self._swa_v2p_table is not None
|
||||
else None
|
||||
)
|
||||
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._full_v2p_table,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_full,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
if out_swa is not None:
|
||||
build_kv_read_table(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
v2p=self._swa_v2p_table,
|
||||
page_size=self.page_size,
|
||||
max_pages=width,
|
||||
out=out_swa,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
return KVIndexTable(
|
||||
ids=out_full,
|
||||
row_ids=self._rows[:bs],
|
||||
row_stride=out_full.stride(0),
|
||||
entry_page_size=self.page_size,
|
||||
is_translated=True,
|
||||
sliding_window_ids=out_swa,
|
||||
)
|
||||
|
||||
def fill_read_table(
|
||||
self,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
sliding_window_out: Optional[torch.Tensor] = None,
|
||||
seq_len_delta: int = 0,
|
||||
) -> None:
|
||||
"""`build_index_table(into=...)` for a caller that owns a bare block
|
||||
table rather than a KVReadTables: the page-table consumers read that
|
||||
table directly, its rows already being the index table's rows.
|
||||
|
||||
`sliding_window_out` fills the swa twin in the same pass, for a hybrid
|
||||
model whose kernels take two block tables.
|
||||
"""
|
||||
# `reads_are_translated`, not `is_translating`: under DCP the builder
|
||||
# returns the passthrough view and writes nothing, so `is_translating`
|
||||
# would let a caller keep a stale table and never hear about it.
|
||||
assert self.reads_are_translated, (
|
||||
"KVIndexTranslator.fill_read_table cannot fill a page table when "
|
||||
"reads stay virtual (a non-unified pool, or DCP, where the caller "
|
||||
"must select this rank's share itself)"
|
||||
)
|
||||
assert sliding_window_out is None or self._swa_v2p_table is not None, (
|
||||
"KVIndexTranslator.fill_read_table: asked for a sliding-window "
|
||||
"table on a pool with no swa sub-pool"
|
||||
)
|
||||
self.build_index_table(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
into=KVReadTables(full=out, sliding_window=sliding_window_out),
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
|
||||
"""Eager per-batch view, memoized in one slot keyed by batch identity
|
||||
so multi-consumer metadata builds share a build. The next batch
|
||||
replaces the slot; consumers only read during their own build. The
|
||||
captured path does not memoize -- it refreshes its buffers per
|
||||
replay."""
|
||||
memo = self._index_table_memo
|
||||
if memo is not None and memo[0]() is forward_batch:
|
||||
return memo[1]
|
||||
max_pages = None
|
||||
if self.is_translating:
|
||||
# `seq_lens_cpu` is a non-None but STALE slice on a gpu_only
|
||||
# batch; `seq_lens_sum` is the signal that it is live. A stale max
|
||||
# under-sizes the table and the tail then reads as the sink.
|
||||
slc = forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is not None
|
||||
and slc is not None
|
||||
and slc.numel() > 0
|
||||
):
|
||||
max_seq = int(slc.max())
|
||||
else:
|
||||
max_seq = self.req_to_token.shape[1]
|
||||
max_pages = max(-(-max_seq // self.page_size), 1)
|
||||
view = self.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
max_pages=max_pages,
|
||||
)
|
||||
self._index_table_memo = (weakref.ref(forward_batch), view)
|
||||
return view
|
||||
return self._reads_translated(IdSpaceKind.FULL)
|
||||
|
||||
@property
|
||||
def full_v2p_table(self) -> Optional[torch.Tensor]:
|
||||
@@ -426,43 +558,11 @@ class KVIndexTranslator:
|
||||
pool needs no translation.
|
||||
|
||||
For the DCP page-table builders, whose gather is over a rank's cyclic
|
||||
slice rather than a row prefix, so `build_index_table` cannot serve
|
||||
them.
|
||||
slice rather than a row prefix, so a plan's row-prefix table cannot
|
||||
serve them.
|
||||
"""
|
||||
return self._full_v2p_table
|
||||
|
||||
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
|
||||
"""Whole-sequence-verify view: every row's live prefix widened by
|
||||
`seq_len_delta` columns (the drafts are read back from the pool).
|
||||
Not memoized -- verify is one consumer, and the batch's memoized
|
||||
prefix table stays valid for the others.
|
||||
|
||||
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
|
||||
carries `seq_lens_cpu` already expanded by the window and keeps the
|
||||
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
|
||||
lens would count the window twice. The width never exceeds the
|
||||
`req_to_token` row it is read from."""
|
||||
max_pages = None
|
||||
if self.is_translating:
|
||||
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
|
||||
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
|
||||
slc = live if live is not None else forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is not None
|
||||
and slc is not None
|
||||
and slc.numel() > 0
|
||||
):
|
||||
max_seq = int(slc.max()) + seq_len_delta
|
||||
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
|
||||
else:
|
||||
max_pages = row_pages
|
||||
return self.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens=forward_batch.seq_lens,
|
||||
max_pages=max_pages,
|
||||
seq_len_delta=seq_len_delta,
|
||||
)
|
||||
|
||||
def bind_and_verify_backends(self, backends) -> None:
|
||||
"""Boot: make every reachable backend carry THIS translator.
|
||||
|
||||
@@ -481,84 +581,13 @@ class KVIndexTranslator:
|
||||
"copy, not build its own."
|
||||
)
|
||||
|
||||
# -- write loc (phase 1; phase 2 lives in build_index_table) ----------------
|
||||
def bind_own_plan(self, forward_batch) -> None:
|
||||
"""Bind a batch built outside `ForwardBatch.init_new` that is its own
|
||||
iteration to a plan of its own fields (`own_plan`). A forward that
|
||||
shares its iteration's window takes that plan instead."""
|
||||
self.own_plan(forward_batch).bind(forward_batch, self)
|
||||
|
||||
def rebind_write_loc(self, forward_batch) -> None:
|
||||
"""Phase 1 of the WRITE contract: translate the batch's write loc to
|
||||
FULL-side physical ids exactly once, at ForwardBatch construction, and
|
||||
mark it physical. On non-unified pools the allocation is already
|
||||
physical, so only the mark is set.
|
||||
|
||||
REBIND, never mutate: the translate returns a FRESH tensor, so the
|
||||
ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept /
|
||||
in-flight machinery that reads it. The pre-translate tensor stays on
|
||||
the batch for `fill_capture_write_loc`.
|
||||
"""
|
||||
self._index_table_memo = None
|
||||
if forward_batch.out_cache_loc is None:
|
||||
return
|
||||
if self.is_translating:
|
||||
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
|
||||
forward_batch.out_cache_loc = self._translate_write_full(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
forward_batch.out_cache_loc_is_physical = True
|
||||
|
||||
def fill_capture_write_loc(
|
||||
self,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
forward_batch,
|
||||
width: Optional[int] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Translate this batch's WRITE loc straight into ``out``, a backend's
|
||||
capture-stable buffer, and return the live ``[:n]`` view. One launch
|
||||
fills the live prefix and clears the tail a shorter replay leaves;
|
||||
None when this pool needs no translation.
|
||||
|
||||
Must run at metadata-init time: `out` is reused every step, so filling
|
||||
it sooner would race a still-pending previous step under overlap
|
||||
scheduling.
|
||||
"""
|
||||
if not self.is_translating:
|
||||
return None
|
||||
virtual = forward_batch.out_cache_loc_virtual
|
||||
if virtual is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
if loc is None:
|
||||
return None
|
||||
# The runner builds the capture batch outside `init_new`, so no
|
||||
# rebind marked its virtual source; bake it holding sink ids.
|
||||
width = int(loc.numel()) if width is None else int(width)
|
||||
out[:width].zero_()
|
||||
return out[: int(loc.numel())]
|
||||
n = int(virtual.numel())
|
||||
width = n if width is None else int(width)
|
||||
buf = out[:width]
|
||||
self._translate_write_full(virtual, out=buf, out_width=width)
|
||||
return buf[:n]
|
||||
|
||||
def sliding_window_write_loc_for(
|
||||
self, out_cache_loc: Optional[torch.Tensor]
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""This batch's sliding-window write loc, or None when there is no loc
|
||||
this forward or the pool has no sliding-window id space."""
|
||||
if out_cache_loc is None or self._swa_write_loc_from_full is None:
|
||||
return None
|
||||
return self._swa_write_loc_from_full(out_cache_loc)
|
||||
|
||||
def _swa_write_loc_unified(self, full_loc: torch.Tensor) -> torch.Tensor:
|
||||
"""Sliding-window write loc, derived pointwise from FULL-side physical
|
||||
ids (phase 2 of the write contract).
|
||||
"""
|
||||
ps = self.page_size
|
||||
offset = full_loc % ps # == virtual_token % page_size
|
||||
# An unmapped physical page reads back as -1; clamp it rather than let
|
||||
# the gather wrap onto the v2p table's last element.
|
||||
virt_page = self._full_p2v_table[full_loc // ps].clamp_(min=0)
|
||||
return (self._swa_v2p_table[virt_page] * ps + offset).clamp_(min=0)
|
||||
|
||||
# -- token-level translate surface (the mixin / local-attn consumers) ------
|
||||
# -- DCP read production ---------------------------------------------------
|
||||
|
||||
@property
|
||||
def needs_read_translate(self) -> bool:
|
||||
@@ -575,14 +604,6 @@ class KVIndexTranslator:
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
if dcp_size > 1:
|
||||
widened_ids = widened_ids // dcp_size
|
||||
return self.translate_full_attn_ids(widened_ids)
|
||||
|
||||
def translate_full_attn_ids(
|
||||
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""Virtual token ids -> physical full-attention ids (the identity
|
||||
when no translation is needed, so callers never branch)."""
|
||||
if not self.is_translating:
|
||||
assert out is None, "passthrough translate takes no out="
|
||||
return kv_indices
|
||||
return self._translate_full(kv_indices, out=out)
|
||||
return widened_ids
|
||||
return self._translate_full(widened_ids)
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""One scheduler iteration's KV slot ids, per sub-pool.
|
||||
|
||||
Under the unified pool a KV slot has a virtual id (what the scheduler allocates
|
||||
and `req_to_token` stores) and, in each sub-pool, a physical id (what that
|
||||
sub-pool's kernels index). Every forward of one iteration -- the target's, and
|
||||
a fused draft's, which lives in the target's pages -- writes the same slots and
|
||||
reads the same rows, and compaction does not move a page while the iteration's
|
||||
forwards are in flight. So:
|
||||
|
||||
* The write window is translated once per sub-pool, by the plan: the
|
||||
full-attention ids when the plan is built, another sub-pool's on first
|
||||
use (`write_ids`). Every writer binds them.
|
||||
* A read is translated exactly once, only by the translator's read
|
||||
primitives (`KVIndexTranslator`); a consumer never translates.
|
||||
* The read table (`read_table`) exists only once a table reader asks for
|
||||
it, built into that reader's own capture-stable table when it is the
|
||||
first. Later readers take it; earlier ones translate in their own gather.
|
||||
* A sub-pool is named by its `IdSpace`; one that indexes virtual ids as they
|
||||
are (a static pool's, a private draft pool's) translates nothing.
|
||||
|
||||
One plan per iteration: `ForwardBatch.init_new` builds it for a lone forward, a
|
||||
speculative worker as soon as the iteration's slots are allocated (shared by
|
||||
its draft and target forwards and direct KV writers), and a graph runner for a
|
||||
batch over its own buffers (`write_slots`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Hashable, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_index_translator import (
|
||||
KVIndexTable,
|
||||
KVIndexTranslator,
|
||||
)
|
||||
|
||||
|
||||
class IdSpaceKind(enum.Enum):
|
||||
"""A sub-pool of a runner's KV pool, as the consumers that index it name
|
||||
it."""
|
||||
|
||||
# Full-attention layers: the pool's own token ids.
|
||||
FULL = enum.auto()
|
||||
# The sliding-window layers of a hybrid pool: the same token ids through
|
||||
# the sliding-window sub-pool's table.
|
||||
SLIDING_WINDOW = enum.auto()
|
||||
|
||||
|
||||
class IdSpace(msgspec.Struct, frozen=True):
|
||||
"""How one sub-pool names the iteration's slots.
|
||||
|
||||
``write`` maps the iteration's virtual token ids to this sub-pool's ids
|
||||
(None: they index it as they are). ``read_v2p`` is the page table its reads
|
||||
go through (None: reads stay in `req_to_token`, for the consumer to
|
||||
finish). Runners whose pools share a sub-pool share its ``key``, and with
|
||||
it a plan's ids for it."""
|
||||
|
||||
key: Hashable
|
||||
write: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
read_v2p: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def window_read_extent(forward_mode, spec_info, write_ids, batch_size: int) -> int:
|
||||
"""How far past ``seq_lens`` a forward reads: the window it writes, when
|
||||
its ``seq_lens`` do not count that window yet. A verify reads at most
|
||||
``draft_token_num`` past them (a ragged verify writes fewer for some rows);
|
||||
a draft decode reads back its own earlier steps; a draft extend's lengths
|
||||
take its window only after its batch is built. Zero otherwise."""
|
||||
if spec_info is None or write_ids is None or not batch_size:
|
||||
return 0
|
||||
if forward_mode.is_target_verify():
|
||||
return spec_info.draft_token_num
|
||||
if forward_mode.is_decode() or forward_mode.is_draft_extend_v2():
|
||||
return write_ids.numel() // batch_size
|
||||
return 0
|
||||
|
||||
|
||||
class TokenSpan(msgspec.Struct, frozen=True):
|
||||
"""A contiguous run ``[start, stop)`` of a write window's flat tokens (one
|
||||
half of a two-batch-overlap split), which a reader takes as a view; a
|
||||
``slice`` in `Cols` names columns of every row instead."""
|
||||
|
||||
start: int
|
||||
stop: int
|
||||
|
||||
|
||||
# Part of a `[batch, window]` write window: a slice of its columns, a
|
||||
# contiguous run of its flat tokens, or a flat index into it (-1: the sink).
|
||||
Cols = Union[slice, TokenSpan, torch.Tensor]
|
||||
|
||||
|
||||
def pad_with_sink(ids: Optional[torch.Tensor], n: int) -> Optional[torch.Tensor]:
|
||||
"""``ids`` padded to ``n`` entries with the sink (id 0 in every space), for
|
||||
a batch whose write ids cover more lanes than its plan's (a padded batch,
|
||||
a captured graph's buffer)."""
|
||||
if ids is None or ids.shape[0] >= n:
|
||||
return ids
|
||||
padded = ids.new_zeros(n)
|
||||
padded[: ids.shape[0]].copy_(ids)
|
||||
return padded
|
||||
|
||||
|
||||
class KVLocPlan:
|
||||
"""The ids of one iteration, in every sub-pool, each computed once."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source: KVIndexTranslator,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self._source = source
|
||||
self.req_pool_indices = req_pool_indices
|
||||
# Committed lengths when the plan is built; every read this iteration
|
||||
# stays within `seq_lens + read_extent` (the window it writes).
|
||||
self.seq_lens = seq_lens
|
||||
self.seq_lens_cpu = seq_lens_cpu
|
||||
self.read_extent = read_extent
|
||||
full = source.space(IdSpaceKind.FULL)
|
||||
if write_slots is not None:
|
||||
# A runner's own write buffer (graph capture, warmup): already in
|
||||
# its pool's ids, naming the sink until a replay fills it, and kept
|
||||
# by address by a captured graph.
|
||||
assert write_virtual is None
|
||||
self.write_virtual = None if full.write is not None else write_slots
|
||||
self.write_physical = write_slots
|
||||
else:
|
||||
# Aliases the ScheduleBatch's tensor, which stays virtual for the
|
||||
# radix tree, the accept path and lazy compaction's in-flight write
|
||||
# set.
|
||||
self.write_virtual = write_virtual
|
||||
# FIXME: a speculative iteration builds its plan before its draft
|
||||
# runs, so this launch sits on the host path ahead of the draft's
|
||||
# graph, idle GPU time at low concurrency. Translate the window
|
||||
# inside the iteration's first captured forward instead, into a
|
||||
# capture-stable buffer the plan owns, which every later forward
|
||||
# and direct writer reads -- once per sub-pool, the sliding-window
|
||||
# ids (derived in `write_ids`) included.
|
||||
self.write_physical = (
|
||||
full.write(write_virtual)
|
||||
if full.write is not None and write_virtual is not None
|
||||
else write_virtual
|
||||
)
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
|
||||
full.key: self.write_physical
|
||||
}
|
||||
self._read_tables: Dict[Hashable, KVIndexTable] = {}
|
||||
|
||||
# -- writes ----------------------------------------------------------------
|
||||
|
||||
def bind(self, batch, reader: KVIndexTranslator, *, cols: Optional[Cols] = None):
|
||||
"""Give ``batch`` (a ForwardBatch or a stand-in) this plan, the part of
|
||||
the window it writes (``cols``), and its full-attention write ids in
|
||||
`reader`'s pool. The one way a forward gets its write ids; another
|
||||
sub-pool's come from `KVIndexTranslator.write_ids`."""
|
||||
batch.kv_loc_plan = self
|
||||
batch.kv_loc_cols = cols
|
||||
batch.out_cache_loc = self.write_ids(reader, cols=cols)
|
||||
batch.out_cache_loc_virtual = (
|
||||
self.virtual_write_ids(cols=cols)
|
||||
if self.is_translated_for(reader)
|
||||
else None
|
||||
)
|
||||
batch.out_cache_loc_is_physical = batch.out_cache_loc is not None
|
||||
|
||||
def bind_replay(self, batch, reader: KVIndexTranslator, *, slots: torch.Tensor):
|
||||
"""`bind`, for the batch a captured graph replays this iteration's
|
||||
forward with. Its write ids are ``slots``, the runner's capture-stable
|
||||
buffer that already holds this plan's write ids and names the sink past
|
||||
them. The virtual mirror stays with the live batch."""
|
||||
self.bind(batch, reader)
|
||||
batch.out_cache_loc = slots
|
||||
batch.out_cache_loc_virtual = None
|
||||
|
||||
def write_ids(
|
||||
self,
|
||||
reader: KVIndexTranslator,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
cols: Optional[Cols] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""The write window as `reader`'s ``kind`` sub-pool indexes it, or None
|
||||
when its pool has no such sub-pool. ``cols`` selects part of a
|
||||
``[batch, window]`` window, for a forward that writes only part of it:
|
||||
a slice of columns (a draft step ahead of the verify window it opens),
|
||||
or a flat index into the window, -1 naming the sink (a ragged verify's
|
||||
packed rows)."""
|
||||
space = reader.space(kind)
|
||||
if space is None:
|
||||
return None
|
||||
if space.key not in self._write_ids:
|
||||
self._write_ids[space.key] = self._window_in(space)
|
||||
return self._cols(self._write_ids[space.key], cols)
|
||||
|
||||
def _window_in(self, space: IdSpace) -> Optional[torch.Tensor]:
|
||||
if self.write_virtual is None:
|
||||
# No window this iteration, or a runner's own slots, which name
|
||||
# the sink in every other sub-pool too.
|
||||
if self.write_physical is None:
|
||||
return None
|
||||
return torch.zeros_like(self.write_physical)
|
||||
if space.write is None:
|
||||
return self.write_virtual
|
||||
return space.write(self.write_virtual)
|
||||
|
||||
def virtual_write_ids(self, *, cols: Optional[Cols] = None):
|
||||
"""The same columns in the virtual space: the ids the scheduler's
|
||||
bookkeeping (lazy compaction's in-flight write set, the radix tree)
|
||||
names."""
|
||||
return self._cols(self.write_virtual, cols)
|
||||
|
||||
def _cols(self, ids: Optional[torch.Tensor], cols: Optional[Cols]):
|
||||
if ids is None or cols is None:
|
||||
return ids
|
||||
if isinstance(cols, TokenSpan):
|
||||
return ids[cols.start : cols.stop]
|
||||
if isinstance(cols, torch.Tensor):
|
||||
return torch.where(cols >= 0, ids[cols.clamp(min=0)], 0)
|
||||
bs = int(self.req_pool_indices.numel())
|
||||
if bs == 1:
|
||||
return ids[cols]
|
||||
return ids.view(bs, -1)[:, cols].reshape(-1)
|
||||
|
||||
def cols_slice(self, cols: Optional[Cols], tokens: slice) -> Cols:
|
||||
"""The ``tokens`` of a forward that writes ``cols``: the columns of
|
||||
one half of a batch split in two (two-batch overlap). A contiguous
|
||||
half stays a `TokenSpan`; only columns spread over the rows become a
|
||||
flat index into the window. Tokens past the window are left out; the
|
||||
consumer pads them with the sink."""
|
||||
if isinstance(cols, torch.Tensor):
|
||||
return cols[tokens]
|
||||
window = (
|
||||
self.write_virtual
|
||||
if self.write_virtual is not None
|
||||
else self.write_physical
|
||||
)
|
||||
if cols is None or isinstance(cols, TokenSpan):
|
||||
base = 0 if cols is None else cols.start
|
||||
end = window.numel() if cols is None else cols.stop
|
||||
start, stop, step = tokens.indices(end - base)
|
||||
if step == 1:
|
||||
return TokenSpan(base + start, base + max(start, stop))
|
||||
index = torch.arange(window.numel(), device=window.device)
|
||||
return self._cols(index, cols)[tokens]
|
||||
|
||||
def reads_from(
|
||||
self,
|
||||
source: KVIndexTranslator,
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
read_extent: int,
|
||||
) -> KVLocPlan:
|
||||
"""This plan's write ids, with reads planned over ``source``'s own
|
||||
`req_to_token` rows and lengths -- a draft that reads a compact table
|
||||
of its own while writing the iteration's window. The write ids are
|
||||
shared, not translated again."""
|
||||
plan = KVLocPlan(
|
||||
source=source,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=None,
|
||||
read_extent=read_extent,
|
||||
)
|
||||
plan.write_virtual = self.write_virtual
|
||||
plan.write_physical = self.write_physical
|
||||
plan._write_ids = self._write_ids
|
||||
return plan
|
||||
|
||||
# -- reads -----------------------------------------------------------------
|
||||
|
||||
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
|
||||
"""Whether a reader has had the ``kind`` table built yet."""
|
||||
return self._source.space(kind).key in self._read_tables
|
||||
|
||||
def read_table(
|
||||
self,
|
||||
*,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
rows: Optional[int] = None,
|
||||
into: Optional[torch.Tensor] = None,
|
||||
) -> KVIndexTable:
|
||||
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
|
||||
when a table reader first asks (in place in ``into``, that reader's
|
||||
capture-stable table, when given) and shared by every later reader.
|
||||
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
|
||||
sink, appended by copying, never by translating again. Where reads stay
|
||||
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
|
||||
space = self._source.space(kind)
|
||||
table = self._read_tables.get(space.key)
|
||||
if table is None or (
|
||||
table.is_translated and rows is not None and table.ids.shape[0] < rows
|
||||
):
|
||||
table = self._source.build_iteration_table(
|
||||
self, space, rows=rows, previous=table, into=into
|
||||
)
|
||||
self._read_tables[space.key] = table
|
||||
else:
|
||||
assert into is None, "a destination goes to the table's first build"
|
||||
if rows is not None and table.ids.shape[0] > rows:
|
||||
# Built wider, for a captured graph's padded lanes: this reader's
|
||||
# kernels size their batch by the table's rows.
|
||||
table = msgspec.structs.replace(
|
||||
table, ids=table.ids[:rows], row_ids=table.row_ids[:rows]
|
||||
)
|
||||
return table
|
||||
|
||||
def is_read_by(self, reader: KVIndexTranslator) -> bool:
|
||||
"""Whether `reader` reads this plan's tables: it indexes the plan's
|
||||
physical ids and gathers the same `req_to_token` rows. A reader with a
|
||||
table of its own (a draft's compact `req_to_token`) plans its own
|
||||
reads."""
|
||||
return (
|
||||
self.is_translated_for(reader)
|
||||
and reader.req_to_token is self._source.req_to_token
|
||||
)
|
||||
|
||||
def is_translated_for(self, reader: KVIndexTranslator) -> bool:
|
||||
"""Whether `reader`'s pool indexes this plan's physical ids: the
|
||||
target's, and a fused draft's, which lives in the target's full
|
||||
sub-pool. A pass-through reader (a static pool, a private draft pool)
|
||||
indexes the virtual ids."""
|
||||
full = reader.space(IdSpaceKind.FULL)
|
||||
return full.write is not None and full.key == self._full_key
|
||||
@@ -5,45 +5,23 @@ The nesting, outermost first:
|
||||
sub-pool a named pool inside the unified pool, with its own grow direction
|
||||
and frontier. HOST is the role it plays when its entries also
|
||||
carry draft bytes; only "full" hosts a draft here.
|
||||
slot one token entry of that sub-pool. The draft's KV becomes extra
|
||||
parts of every slot, after the host's own parts at
|
||||
`draft_offset_in_entry()`, so ONE slot id per token covers target
|
||||
and draft: one allocation, one free, one whole-page relocation
|
||||
carry both.
|
||||
slot one token entry of that sub-pool. The draft's KV is extra parts of
|
||||
every slot, after the host's own parts, so ONE slot id per token
|
||||
covers target and draft: one allocation, one free, one whole-page
|
||||
relocation carry both.
|
||||
region the draft geometry inside the host's entries. A region is not a
|
||||
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
|
||||
lane one position in the region's layer dimension -- one draft layer of
|
||||
one runner, present in every slot.
|
||||
|
||||
Runners are the draft's execution copies, and the two head shapes place them
|
||||
oppositely. A replicated head gives EVERY runner all the draft's layers, so
|
||||
each needs its own lanes or the runners clobber each other's KV (3 runners x
|
||||
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
|
||||
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
|
||||
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
|
||||
either direction. Each runner owns a contiguous run of lanes, and the runs
|
||||
tile the region in runner order.
|
||||
A lane is not a layer. Runners are the draft's execution copies: a
|
||||
replicated head gives EVERY runner all the draft's layers, on lanes of its own
|
||||
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
|
||||
while a per-depth head (one block per predicted position) gives each runner
|
||||
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
|
||||
contiguous run of lanes, and the runs tile the region in runner order.
|
||||
|
||||
Three phases, three moments in boot, and what carries each:
|
||||
|
||||
PLAN on the TARGET, before any memory exists
|
||||
KVCacheConfigurator._fused_draft_decision()
|
||||
draft_kv_profile(draft_model_config) what the checkpoint asks for
|
||||
place_fused_draft(profile, runners) admit, or decline with a reason
|
||||
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
|
||||
|
||||
CARVE as the byte buffer is cut
|
||||
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
|
||||
init_unified_*_pools(fused_draft=...)
|
||||
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
|
||||
UnifiedKVPool.build_dense_draft_views()
|
||||
|
||||
BIND on each DRAFT runner, built after the target
|
||||
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
|
||||
bind_fused_draft(...) -> UnifiedDraftKVPool
|
||||
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
|
||||
|
||||
A draft whose layers need another sub-pool kind declines to a private pool.
|
||||
A draft whose layers need another sub-pool kind declines fusion as a whole.
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
@@ -57,8 +35,8 @@ from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
|
||||
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
|
||||
sub-pool. The draft is a separate checkpoint, so its head geometry and
|
||||
layer count differ from the host's; its K and V rows are two more parts of
|
||||
the host entry, indexed by the host's physical token id."""
|
||||
layer count can differ from the host's; its K and V rows are two more parts
|
||||
of the host entry, indexed by the host's physical token id."""
|
||||
|
||||
lane_num: int
|
||||
head_num: int
|
||||
@@ -174,7 +152,6 @@ def draft_state_layer_num(draft_model_config) -> int:
|
||||
mambaish = mambaish_config(draft_model_config)
|
||||
if mambaish is None:
|
||||
return 0
|
||||
# Optional HF attribute: only conv-chain MTP configs declare it.
|
||||
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
|
||||
return 0
|
||||
return len(mambaish.mamba2_cache_params.layers)
|
||||
@@ -217,8 +194,8 @@ def draft_kv_profile(
|
||||
|
||||
|
||||
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
|
||||
private pool. Neither means fusion simply does not apply."""
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
|
||||
fuse. Neither means fusion does not apply."""
|
||||
|
||||
placement: Optional[FusedDraftPlacement] = None
|
||||
declined: Optional[str] = None
|
||||
@@ -253,8 +230,8 @@ def place_fused_draft(
|
||||
) -> FusedDraftDecision:
|
||||
"""Assign every draft layer of every runner to the host sub-pool whose
|
||||
lifetime covers what the layer reads: a full-attention layer rides in
|
||||
``"full"``. A layer kind no host arm serves declines the whole draft to
|
||||
its private pool."""
|
||||
``"full"``. A layer kind no host sub-pool serves declines the whole
|
||||
draft."""
|
||||
counts, reason = _runner_layer_counts(profile, num_runners)
|
||||
if counts is None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
|
||||
@@ -1871,26 +1871,21 @@ class KVWriteLoc:
|
||||
|
||||
All location info lives here (in the attention metadata), NOT in the pool:
|
||||
- ``loc``: the generic per-token write location (``out_cache_loc``).
|
||||
PHYSICAL on every pool: by allocation on non-unified pools,
|
||||
rebound at ForwardBatch construction (``rebind_write_loc``) on
|
||||
the unified pool.
|
||||
PHYSICAL on every pool: by allocation on non-unified pools, the
|
||||
iteration's plan's write ids on the unified pool (``KVLocPlan.bind``).
|
||||
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
|
||||
otherwise); under the unified pool the translator derives it from the
|
||||
same rebound loc (``sliding_window_write_loc_for``).
|
||||
- ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the
|
||||
construction-time rebind it is the SAME id space as ``loc``, so pools
|
||||
fall back to ``loc`` when it is ``None`` -- only triton's captured path
|
||||
still passes its capture-stable
|
||||
``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a
|
||||
same-space alias slated for collapse).
|
||||
otherwise): the same plan's write ids in the sliding-window sub-pool
|
||||
(``KVIndexTranslator.write_ids``).
|
||||
- ``full_loc``: OPTIONAL full-attention-sub-pool location, in the same id
|
||||
space as ``loc``; pools fall back to ``loc`` when it is ``None``.
|
||||
|
||||
``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved
|
||||
loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``);
|
||||
``loc`` is the generic fallback. Bundling them lets a backend issue one
|
||||
``set_kv_buffer`` call regardless of pool type.
|
||||
|
||||
``physical`` marks the locs as physical token ids: the batch's write loc
|
||||
after ``rebind_write_loc``, or ids a backend translated itself. A unified
|
||||
``physical`` marks the locs as physical token ids: the write ids a plan
|
||||
bound to the batch, or ids produced separately and marked so. A unified
|
||||
pool's write door refuses a loc not marked physical; other pools do not
|
||||
check.
|
||||
"""
|
||||
@@ -1910,9 +1905,9 @@ class KVWriteLoc:
|
||||
) -> KVWriteLoc:
|
||||
"""The batch's ``out_cache_loc`` as a write loc, carrying the batch's
|
||||
physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under
|
||||
the same mark, so it must be derived from that rebound loc (as
|
||||
``sliding_window_write_loc_for`` does); a loc produced separately
|
||||
states its own mark with ``KVWriteLoc(loc, physical=...)``."""
|
||||
the same mark, so it must come from the same plan (as
|
||||
``KVIndexTranslator.write_ids`` does); a loc produced separately states
|
||||
its own mark with ``KVWriteLoc(loc, physical=...)``."""
|
||||
return cls(
|
||||
forward_batch.out_cache_loc,
|
||||
swa_loc,
|
||||
@@ -2095,9 +2090,9 @@ class KVCache(abc.ABC):
|
||||
if self.requires_physical_write_loc and not write_loc_is_physical(loc_info):
|
||||
raise ValueError(
|
||||
f"{where}: write loc is not marked physical. Hand the pool "
|
||||
"KVWriteLoc.for_batch(forward_batch) after "
|
||||
"KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, "
|
||||
"physical=True) for ids translated separately."
|
||||
"KVWriteLoc.for_batch(forward_batch) for a batch bound to a "
|
||||
"plan (KVLocPlan.bind), or KVWriteLoc(loc, physical=True) for "
|
||||
"ids translated separately."
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
@@ -4734,7 +4729,7 @@ class MLATokenToKVPool(KVCache):
|
||||
|
||||
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
|
||||
# False: this pool takes a WIDENED loc. The unified pool resolves it in
|
||||
# `KVIndexTranslator.rebind_write_loc` and flips this.
|
||||
# the plan's write translation (`translate_write_loc`) and flips this.
|
||||
write_loc_is_dcp_resolved = False
|
||||
|
||||
@property
|
||||
|
||||
@@ -15,15 +15,11 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
|
||||
"""Dense draft KV over the draft parts of one host sub-pool's entries.
|
||||
|
||||
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
|
||||
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
|
||||
``layer_lanes`` maps each of this runner's layer ids to its region lane.
|
||||
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
|
||||
produced by the allocator's translate (the id-space choke point binds it,
|
||||
see KVIndexTranslator); the pool exposes `host_allocator` for that
|
||||
binding. Relocation needs no method here: compaction moves whole page
|
||||
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
|
||||
raises so a stray per-slot move fails loudly instead of corrupting the
|
||||
fused layout.
|
||||
every slot of the host sub-pool; ``layer_lanes`` maps each of this
|
||||
runner's layer ids to its region lane. Locs are the target's PHYSICAL
|
||||
token ids, translated through `host_allocator`. Compaction moves whole
|
||||
page envelopes on the HOST pool, draft bytes included, so `move_kv_cache`
|
||||
raises rather than let a per-slot move corrupt the fused layout.
|
||||
"""
|
||||
|
||||
requires_physical_write_loc = True
|
||||
|
||||
@@ -149,8 +149,7 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
|
||||
|
||||
The slot stride stays the one entry, so host and draft views are indexed
|
||||
by the same physical token id; only their offsets differ. `draft_region
|
||||
is None` keeps the layout byte-identical to the unfused one.
|
||||
by the same physical token id; only their offsets differ.
|
||||
"""
|
||||
|
||||
head_num: int
|
||||
@@ -930,8 +929,9 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
# Lifetime owned by UnifiedKVPool; do not delete the views.
|
||||
pass
|
||||
|
||||
# `rebind_write_loc` already collapsed the widened id and sent the rows this
|
||||
# rank does not own to the padding sink.
|
||||
# The plan's write translation (`translate_write_loc`) already collapsed
|
||||
# the widened id and sent the rows this rank does not own to the padding
|
||||
# sink.
|
||||
write_loc_is_dcp_resolved = True
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
|
||||
@@ -814,7 +814,6 @@ class CPUGraphRunner:
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -832,6 +831,7 @@ class CPUGraphRunner:
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
global_forward_mode=self.capture_forward_mode,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
# Wrap all forward calls with capture_with_skip_cross_attention so that
|
||||
# mllama (and any other encoder-decoder model) sees the correct compile-
|
||||
# time constant for skip_cross_attention during tracing.
|
||||
|
||||
@@ -83,6 +83,7 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
):
|
||||
self.prefix_chunk_kv_indices = []
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
translator = get_attn_backend().kv_index_translator
|
||||
for idx in range(self.num_prefix_chunks):
|
||||
chunk_starts = self.prefix_chunk_starts[idx]
|
||||
chunk_seq_lens = self.prefix_chunk_seq_lens[idx]
|
||||
@@ -93,6 +94,18 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
num_chunk_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
if translator.reads_are_translated:
|
||||
# The plan's table already holds the chunk's physical ids.
|
||||
translator.pack_read_stream(
|
||||
self.kv_loc_plan,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=chunk_seq_lens,
|
||||
indptr=chunk_cu_seq_lens,
|
||||
out=chunk_kv_indices,
|
||||
kv_start_idx=chunk_starts,
|
||||
)
|
||||
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
|
||||
continue
|
||||
create_chunked_prefix_cache_kv_indices[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
@@ -107,7 +120,6 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
self.prefix_chunk_starts_cpu[idx],
|
||||
self.prefix_chunk_seq_lens_cpu[idx],
|
||||
)
|
||||
translator = get_attn_backend().kv_index_translator
|
||||
chunk_kv_indices = translator.translate_dcp_read_ids(chunk_kv_indices)
|
||||
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
|
||||
|
||||
@@ -266,19 +278,28 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
device=self.req_pool_indices.device,
|
||||
)
|
||||
kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0)
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
create_flashinfer_kv_indices_triton[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
self.seq_lens,
|
||||
kv_indptr,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.shape[1],
|
||||
)
|
||||
# None on a backend that never bound a translator.
|
||||
# None on a backend that never bound a translator, which reads virtual
|
||||
# ids; otherwise one gather from the iteration's plan (DCP takes its
|
||||
# own branch upstream).
|
||||
src = get_attn_backend().kv_index_translator
|
||||
if src is not None:
|
||||
kv_indices = src.translate_full_attn_ids(kv_indices)
|
||||
if src is None:
|
||||
req_to_token = get_req_to_token_pool().req_to_token
|
||||
create_flashinfer_kv_indices_triton[(self.batch_size,)](
|
||||
req_to_token,
|
||||
self.req_pool_indices,
|
||||
self.seq_lens,
|
||||
kv_indptr,
|
||||
None,
|
||||
kv_indices,
|
||||
req_to_token.shape[1],
|
||||
)
|
||||
else:
|
||||
src.pack_read_stream(
|
||||
self.kv_loc_plan,
|
||||
req_pool_indices=self.req_pool_indices,
|
||||
seq_lens=self.seq_lens,
|
||||
indptr=kv_indptr,
|
||||
out=kv_indices,
|
||||
)
|
||||
self.mha_one_shot_kv_indices = kv_indices
|
||||
return kv_indices
|
||||
|
||||
@@ -43,7 +43,6 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
compute_req_all_ids_info,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_slot_in,
|
||||
@@ -77,6 +76,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
|
||||
@@ -192,21 +192,6 @@ def _elastic_should_preserve_local_token_counts(
|
||||
return uneven_token_count
|
||||
|
||||
|
||||
def _localize_npu_dcp_out_cache_loc(
|
||||
out_cache_loc: torch.Tensor,
|
||||
*,
|
||||
interleave_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map allocator-global NPU DCP slots to this target rank."""
|
||||
parallel = get_parallel()
|
||||
return localize_dcp_indices(
|
||||
out_cache_loc,
|
||||
parallel.dcp_size,
|
||||
parallel.dcp_rank,
|
||||
interleave_size,
|
||||
)
|
||||
|
||||
|
||||
class ForwardMode(IntEnum):
|
||||
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
|
||||
# It is also called "prefill" in common terminology.
|
||||
@@ -532,14 +517,23 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# The original sequence length without being chunked. Qwen-1M related.
|
||||
orig_seq_lens: Optional[torch.Tensor] = None
|
||||
|
||||
# The write loc before `rebind_write_loc` replaced it with physical
|
||||
# ids; a backend re-derives from it into its capture-stable buffer.
|
||||
# This iteration's KV ids, translated once per sub-pool (`KVLocPlan`).
|
||||
# Every forward of the iteration holds the same plan; `out_cache_loc` is
|
||||
# its write window in the full-attention ids this runner's pool indexes,
|
||||
# and a backend that writes another sub-pool asks the plan for its ids
|
||||
# (`KVIndexTranslator.write_ids`).
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
# The part of the plan's window this forward writes (None: all of it).
|
||||
kv_loc_cols: Optional[Cols] = None
|
||||
# The same write ids in the virtual space when `out_cache_loc` holds
|
||||
# translated ones (None on a pool that indexes virtual ids), for the
|
||||
# scheduler's bookkeeping: lazy compaction's in-flight write set, TBO's
|
||||
# split, state capture.
|
||||
out_cache_loc_virtual: Optional[torch.Tensor] = None
|
||||
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
|
||||
# consumed by the Ascend backend for PA_ND block tables. None elsewhere.
|
||||
out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None
|
||||
# Whether `out_cache_loc` holds physical ids: set by
|
||||
# KVIndexTranslator.rebind_write_loc; capture-time batches declare it.
|
||||
# Whether `out_cache_loc` holds physical ids: set by `KVLocPlan.bind`.
|
||||
out_cache_loc_is_physical: bool = False
|
||||
# The indices to track mamba state with
|
||||
mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64
|
||||
@@ -937,9 +931,16 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
return_hidden_states_before_norm: bool,
|
||||
extend_position_info=None,
|
||||
spec_mrope_positions: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
write_cols: Optional[Cols] = None,
|
||||
):
|
||||
# init_new must not mutate the input ScheduleBatch; per-forward
|
||||
# overrides go through explicit keyword arguments.
|
||||
#
|
||||
# `kv_loc_plan`: the iteration's plan when this forward is one of
|
||||
# several over the same slots (a speculative iteration), `write_cols`
|
||||
# the columns of its window this forward writes. Without one, this
|
||||
# forward is its own iteration and builds the plan itself.
|
||||
|
||||
# capture_hidden_mode=None means no override: capture the server's
|
||||
# configured maximum so lower-mode requests can share one graph.
|
||||
@@ -1051,15 +1052,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
spec_info=batch.spec_info,
|
||||
)
|
||||
|
||||
# ScheduleBatch and req_to_token keep allocator-global slot identities.
|
||||
# Preserve that view before exposing rank-local NPU DCP write slots.
|
||||
# ScheduleBatch and req_to_token keep allocator-global slot identities,
|
||||
# and so does the plan's window. An NPU DCP target writes rank-local
|
||||
# slots, which its translator maps the window to at `bind` below; keep
|
||||
# the global view for the consumers that read it.
|
||||
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
|
||||
ret.origin_out_cache_loc = ret.out_cache_loc
|
||||
if ret.out_cache_loc is not None:
|
||||
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
|
||||
ret.out_cache_loc,
|
||||
interleave_size=model_runner.page_size,
|
||||
)
|
||||
ret._maybe_init_non_generation_fields(batch)
|
||||
|
||||
device = model_runner.device
|
||||
@@ -1073,7 +1071,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
device,
|
||||
)
|
||||
|
||||
model_runner.kv_index_translator.rebind_write_loc(ret)
|
||||
translator = model_runner.kv_index_translator
|
||||
if kv_loc_plan is None:
|
||||
kv_loc_plan = translator.own_plan(ret)
|
||||
kv_loc_plan.bind(ret, translator, cols=write_cols)
|
||||
|
||||
if envs.SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.get():
|
||||
hashed = _hash_rids_to_tensor(
|
||||
@@ -2143,6 +2144,8 @@ def build_inner_fb_view(
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
# A caller may hand in another view that does not carry this field.
|
||||
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
|
||||
kv_loc_plan=getattr(forward_batch, "kv_loc_plan", None),
|
||||
kv_loc_cols=getattr(forward_batch, "kv_loc_cols", None),
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
spec_info=forward_batch.spec_info,
|
||||
|
||||
@@ -898,6 +898,7 @@ class ModelRunner:
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
page_size=self.page_size or 1,
|
||||
device=self.device,
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
)
|
||||
|
||||
def max_shared_logits_buffer_rows(self) -> int:
|
||||
|
||||
@@ -662,7 +662,6 @@ class BaseRunner(ABC):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=batch_size,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -698,6 +697,7 @@ class BaseRunner(ABC):
|
||||
global_forward_mode=capture_forward_mode,
|
||||
lora_ids=lora_ids,
|
||||
)
|
||||
mr.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
if buffers.ngram_embedding_info is not None:
|
||||
forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice(
|
||||
|
||||
@@ -189,6 +189,8 @@ def build_replay_fb_view(
|
||||
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
max_seq_len_override=forward_batch.max_seq_len_override,
|
||||
@@ -1005,7 +1007,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.capture_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -1043,6 +1044,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
rids_int=rids_int,
|
||||
bootstrap_room_ids_int=bootstrap_room_ids_int,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
# Trip the coordinator so the hisparse code path is captured into the
|
||||
# graph; backends read it from self.model_runner.hisparse_coordinator.
|
||||
@@ -1118,9 +1120,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self._post_process_after_profile(prof)
|
||||
self._profiler = None
|
||||
|
||||
# No pool-side pin to clear: the captured full-physical write loc rides the
|
||||
# backend's `ForwardMetadata.out_cache_loc_full_physical` (-> KVWriteLoc.full_loc).
|
||||
|
||||
def _capture_one_stream(self, stream_idx: Optional[int] = None) -> None:
|
||||
avail_mem = get_available_gpu_memory(
|
||||
self.model_runner.device,
|
||||
@@ -1271,9 +1270,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if (c := self.model_runner.canary_manager) is not None
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
# Full-physical write loc lives in the attention metadata (the backend's
|
||||
# `out_cache_loc_full_physical` -> KVWriteLoc.full_loc), so the runner
|
||||
# wires no buffer here. (SWA write loc rides the `swa_out_cache_loc` rail.)
|
||||
|
||||
with canary_ctx:
|
||||
shape_key = self._make_graph_key(
|
||||
|
||||
@@ -1496,7 +1496,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
with torch.device(self.device):
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=_slot("input_ids"),
|
||||
input_embeds=(
|
||||
@@ -1562,6 +1561,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
|
||||
max_seq_len_override=self.max_context_size,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
ngram_manager = self.model_runner.ngram_embedding_manager
|
||||
if ngram_manager.enabled:
|
||||
forward_batch.ngram_embedding_info = NgramEmbeddingInfo.create(
|
||||
@@ -1861,7 +1861,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
static_forward_batch = ForwardBatch(
|
||||
forward_mode=pcg_forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
@@ -1926,6 +1925,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
),
|
||||
max_seq_len_override=self.max_context_size,
|
||||
)
|
||||
forward_batch.kv_loc_plan.bind_replay(
|
||||
static_forward_batch,
|
||||
self.model_runner.kv_index_translator,
|
||||
slots=out_cache_loc,
|
||||
)
|
||||
# The n-gram hasher runs outside the graph and reads this at replay.
|
||||
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
|
||||
static_forward_batch.engram_history = forward_batch.engram_history
|
||||
|
||||
@@ -803,8 +803,8 @@ class DSparkDraftMixin:
|
||||
attn.attn.v_scale,
|
||||
)
|
||||
else:
|
||||
# Translated by the injector (physical by allocation on a plain
|
||||
# pool).
|
||||
# `cache_loc` comes from `TargetHiddenKvInjector.ids_for`,
|
||||
# already in the ids the draft pool indexes.
|
||||
pool.set_kv_buffer(
|
||||
attn.attn,
|
||||
KVWriteLoc(cache_loc, physical=True),
|
||||
|
||||
@@ -481,12 +481,8 @@ class InklingAttention(nn.Module):
|
||||
do_store = do_bf16_store or do_mxfp8_store
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
# SWA sub-pool + the backend's full->SWA translated write location.
|
||||
# SWA sub-pool + the plan's sliding-window write ids (backend buffer).
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
es = q.element_size()
|
||||
@@ -584,10 +580,6 @@ class InklingAttention(nn.Module):
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
is_v2 = forward_batch.forward_mode.is_draft_extend_v2()
|
||||
@@ -691,10 +683,6 @@ class InklingAttention(nn.Module):
|
||||
metadata = get_attn_backend().forward_metadata
|
||||
if do_store and self.is_local:
|
||||
loc = metadata.swa_out_cache_loc
|
||||
elif do_store:
|
||||
loc = getattr(metadata, "out_cache_loc_full_physical", None)
|
||||
if loc is None:
|
||||
loc = forward_batch.out_cache_loc
|
||||
else:
|
||||
loc = forward_batch.out_cache_loc
|
||||
es = q.element_size()
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
@@ -51,6 +52,10 @@ class DFlashVerifyInput(SpecInput):
|
||||
live_seq_lens_cpu: Optional[torch.Tensor] = None
|
||||
# Conservative request-lifetime bound for candidate graph dispatch.
|
||||
candidate_max_seq_len_upper_bound: Optional[int] = None
|
||||
# The iteration's plan when the caller planned the verify window, and the
|
||||
# part of it this verify writes (`KVLocPlan.write_ids`).
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
kv_loc_cols: Optional[Cols] = None
|
||||
|
||||
def __post_init__(self):
|
||||
super().__init__(spec_input_type=SpecInputType.DFLASH_VERIFY)
|
||||
@@ -101,6 +106,8 @@ class DFlashVerifyInput(SpecInput):
|
||||
target_worker.model_runner,
|
||||
capture_hidden_mode=self.capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=self.kv_loc_plan,
|
||||
write_cols=self.kv_loc_cols,
|
||||
)
|
||||
|
||||
can_run_cuda_graph = bool(
|
||||
@@ -133,13 +140,14 @@ class DFlashVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
plan: KVLocPlan,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kv_indices_buf: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
device = req_pool_indices.device
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
@@ -179,14 +187,14 @@ class DFlashVerifyInput(SpecInput):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
|
||||
out=kv_indices,
|
||||
kv_start_idx=kv_start_idx,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
mask = self.custom_mask
|
||||
if mask is not None:
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -1744,6 +1745,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
return out_tokens
|
||||
|
||||
def _draft_write_ids(self, plan: KVLocPlan) -> torch.Tensor:
|
||||
"""The plan's write ids as the draft pool indexes them: the target's
|
||||
physical ids for a fused draft, the virtual ids for a private one."""
|
||||
return plan.write_ids(self.draft_model_runner.kv_index_translator)
|
||||
|
||||
def _append_target_hidden_to_draft_kv_by_loc(
|
||||
self,
|
||||
*,
|
||||
@@ -1845,17 +1851,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
if commit_lens.dtype != torch.int32:
|
||||
commit_lens = commit_lens.to(torch.int32)
|
||||
|
||||
# On a translating pool the translate writes a new tensor, so the
|
||||
# callers' locs stay virtual: the post-verify 2-D buffer is re-read as
|
||||
# virtual ids by the compact req_to_token rebuild. On a plain pool it
|
||||
# returns `cache_loc` itself.
|
||||
translator = self.draft_model_runner.kv_index_translator
|
||||
cache_loc = translator.translate_full_attn_ids(cache_loc)
|
||||
if cache_loc_2d is not None:
|
||||
# Same ids, so reshape the translated copy rather than translate
|
||||
# them a second time.
|
||||
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
|
||||
|
||||
# `cache_loc` arrives in the draft pool's id space (`_draft_write_ids`);
|
||||
# the callers' window buffers stay virtual for the compact
|
||||
# req_to_token rebuild.
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
@@ -2265,7 +2263,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
logits_output, next_token_ids = (
|
||||
batch_output.logits_output,
|
||||
@@ -2316,9 +2316,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
ctx_lens,
|
||||
int(sum(batch.extend_lens)),
|
||||
)
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
self._append_target_hidden_to_draft_kv_by_loc(
|
||||
target_hidden=logits_output.hidden_states,
|
||||
cache_loc=batch.out_cache_loc,
|
||||
cache_loc=self._draft_write_ids(kv_loc_plan),
|
||||
positions=positions,
|
||||
extend_lens=ctx_lens,
|
||||
)
|
||||
@@ -2494,6 +2495,19 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
positions = positions_2d.reshape(-1)
|
||||
verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1)
|
||||
# The iteration's window -- the draft block, the verify and the
|
||||
# target-hidden KV writes all write it -- planned once.
|
||||
kv_loc_plan = self.target_worker.model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=(
|
||||
batch.seq_lens_cpu
|
||||
if batch.seq_lens_cpu is not None
|
||||
else draft_input.nxt_kv_lens_cpu
|
||||
),
|
||||
write_virtual=verify_out_cache_loc,
|
||||
read_extent=block_size,
|
||||
)
|
||||
|
||||
seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs]
|
||||
if self.use_compact_draft_cache:
|
||||
@@ -2555,10 +2569,20 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
),
|
||||
global_num_token_non_padded_cpu=bs * block_size,
|
||||
)
|
||||
# Hand-built draft batch bypasses ForwardBatch.init_new: under the
|
||||
# unified pool the write loc must be rebound to the draft's
|
||||
# kernel-facing ids here (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(forward_batch)
|
||||
# The draft block writes the whole window. A compact draft reads its
|
||||
# own `req_to_token` rows over its own lengths.
|
||||
draft_translator = self.draft_model_runner.kv_index_translator
|
||||
draft_plan = (
|
||||
kv_loc_plan.reads_from(
|
||||
draft_translator,
|
||||
seq_lens=draft_seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
read_extent=block_size,
|
||||
)
|
||||
if self.use_compact_draft_cache
|
||||
else kv_loc_plan
|
||||
)
|
||||
draft_plan.bind(forward_batch, draft_translator)
|
||||
|
||||
if self.selector is not None or self.lilicorr is not None:
|
||||
self._selector_sample = None
|
||||
@@ -2713,6 +2737,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
draft_token_num=int(self.block_size),
|
||||
custom_mask=custom_mask,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
batch.out_cache_loc = verify_out_cache_loc
|
||||
@@ -2877,10 +2902,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
# Consume in this step: every decode graph size shares one aux output,
|
||||
# which the next target forward overwrites (resolve_aux_hidden_states_width).
|
||||
cache_loc = self._draft_write_ids(kv_loc_plan)
|
||||
self._append_target_hidden_to_draft_kv_by_loc(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
cache_loc=verify_out_cache_loc,
|
||||
cache_loc_2d=verify_out_cache_loc_2d,
|
||||
cache_loc=cache_loc,
|
||||
cache_loc_2d=cache_loc.view(verify_out_cache_loc_2d.shape),
|
||||
positions=positions,
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
|
||||
@@ -360,7 +360,7 @@ class DraftBlockProposer:
|
||||
)
|
||||
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
|
||||
# loc to the draft's kernel-facing ids (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(idle_batch)
|
||||
self.draft_model_runner.kv_index_translator.bind_own_plan(idle_batch)
|
||||
self._fill_dp_moe_sync_metadata(idle_batch, batch)
|
||||
with torch.inference_mode():
|
||||
self.draft_model_runner.forward(idle_batch)
|
||||
@@ -435,10 +435,10 @@ class DraftBlockProposer:
|
||||
),
|
||||
global_num_token_non_padded_cpu=draft_num_tokens,
|
||||
)
|
||||
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
|
||||
# loc to the draft's kernel-facing ids (no-op on plain pools).
|
||||
self.draft_model_runner.kv_index_translator.rebind_write_loc(
|
||||
draft_forward_batch
|
||||
verify_window.kv_loc_plan.bind(
|
||||
draft_forward_batch,
|
||||
self.draft_model_runner.kv_index_translator,
|
||||
cols=slice(0, query_token_num),
|
||||
)
|
||||
self._fill_dp_moe_sync_metadata(draft_forward_batch, batch)
|
||||
graph_runner = self.draft_model_runner.decode_cuda_graph_runner
|
||||
|
||||
@@ -5,12 +5,12 @@ import torch
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
BuildCommitInjectLayout,
|
||||
build_unified_commit_inject_layout,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
|
||||
|
||||
@@ -32,6 +32,11 @@ class TargetHiddenKvInjector:
|
||||
self.verify_num_draft_tokens = verify_num_draft_tokens
|
||||
self._block_pos_offsets = block_pos_offsets
|
||||
|
||||
def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor:
|
||||
"""The plan's write ids as the draft pool indexes them: the target's
|
||||
physical ids for a fused draft, the virtual ids for a private one."""
|
||||
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
|
||||
|
||||
def inject_target_hidden(
|
||||
self,
|
||||
*,
|
||||
@@ -70,16 +75,7 @@ class TargetHiddenKvInjector:
|
||||
device=device, dtype=torch.int64, non_blocking=True
|
||||
)
|
||||
|
||||
# On a translating pool the translate writes a new tensor, so the
|
||||
# callers' locs stay virtual; on a plain pool it returns `cache_loc`
|
||||
# itself.
|
||||
translator = self.draft_model_runner.kv_index_translator
|
||||
cache_loc = translator.translate_full_attn_ids(cache_loc)
|
||||
if cache_loc_2d is not None:
|
||||
# Callers pass `cache_loc` viewed [bs, verify_len]: reshape the
|
||||
# translated copy rather than translate the same ids twice.
|
||||
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
|
||||
|
||||
# `cache_loc` arrives in the draft pool's id space (`ids_for`).
|
||||
pool = self.draft_model_runner.token_to_kv_pool
|
||||
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
|
||||
if target_hidden_is_projected:
|
||||
@@ -214,6 +210,7 @@ class TargetHiddenKvInjector:
|
||||
hidden_strided: torch.Tensor,
|
||||
commit_lens: torch.Tensor,
|
||||
bs: int,
|
||||
kv_loc_plan: KVLocPlan,
|
||||
) -> None:
|
||||
stride = self.verify_num_draft_tokens
|
||||
prefix_lens = batch.seq_lens
|
||||
@@ -252,15 +249,7 @@ class TargetHiddenKvInjector:
|
||||
return
|
||||
|
||||
positions_2d = prefix_lens.unsqueeze(1) + self._block_pos_offsets
|
||||
verify_cache_loc = assign_extend_cache_locs_func(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=self.model_runner.req_to_token_pool.req_to_token,
|
||||
start_offset=prefix_lens,
|
||||
end_offset=prefix_lens + stride,
|
||||
batch_size=bs,
|
||||
draft_token_num=stride,
|
||||
device=self.device,
|
||||
)
|
||||
verify_cache_loc = self.ids_for(kv_loc_plan)
|
||||
verify_cache_loc_2d = verify_cache_loc.view(bs, stride)
|
||||
self.inject_target_hidden(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.srt.managers.overlap_utils import (
|
||||
ResolvedConfidence,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
|
||||
from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments
|
||||
@@ -65,6 +66,9 @@ class VerifyWindow(msgspec.Struct, frozen=True):
|
||||
positions_2d: torch.Tensor
|
||||
verify_cache_loc: torch.Tensor
|
||||
verify_cache_loc_2d: torch.Tensor
|
||||
# The iteration's plan of `verify_cache_loc`: the draft block, the verify
|
||||
# and the target-hidden KV writes all take their ids from it.
|
||||
kv_loc_plan: KVLocPlan
|
||||
|
||||
|
||||
class DSparkVerifyPlanner:
|
||||
@@ -827,7 +831,10 @@ def alloc_verify_window(
|
||||
verify_num_draft_tokens: int,
|
||||
block_pos_offsets: torch.Tensor,
|
||||
model_runner,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
) -> VerifyWindow:
|
||||
"""``seq_lens_cpu``: a host bound on ``batch.seq_lens``, which sizes the
|
||||
plan's read table."""
|
||||
prefix_lens = batch.seq_lens
|
||||
verify_w = verify_num_draft_tokens
|
||||
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
|
||||
@@ -845,6 +852,13 @@ def alloc_verify_window(
|
||||
positions_2d=positions_2d,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_cache_loc_2d=verify_cache_loc_2d,
|
||||
kv_loc_plan=model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=verify_cache_loc,
|
||||
read_extent=verify_w,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -312,6 +312,7 @@ class TargetVerifyExecutor:
|
||||
custom_mask=None,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
live_seq_lens_cpu=batch.seq_lens_cpu,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
batch.out_cache_loc = verify_cache_loc
|
||||
seq_lens_cpu_backup = batch.seq_lens_cpu
|
||||
@@ -388,6 +389,7 @@ class TargetVerifyExecutor:
|
||||
hidden_strided=hidden_strided,
|
||||
commit_lens=commit_lens,
|
||||
bs=bs,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
return
|
||||
hidden = logits_output.hidden_states
|
||||
@@ -404,10 +406,11 @@ class TargetVerifyExecutor:
|
||||
state_slot = (
|
||||
batch.req_pool_indices[:bs].view(-1, 1).expand(bs, vlen).reshape(-1)
|
||||
)
|
||||
cache_loc = self.kv_injector.ids_for(verify_window.kv_loc_plan)
|
||||
self.kv_injector.inject_target_hidden(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
cache_loc=verify_window.verify_cache_loc,
|
||||
cache_loc_2d=verify_window.verify_cache_loc_2d,
|
||||
cache_loc=cache_loc,
|
||||
cache_loc_2d=cache_loc.view(verify_window.verify_cache_loc_2d.shape),
|
||||
positions=verify_window.positions_2d.reshape(-1),
|
||||
commit_lens=commit_lens,
|
||||
state_slot=state_slot,
|
||||
@@ -420,6 +423,7 @@ class TargetVerifyExecutor:
|
||||
layout: RaggedVerifyLayout,
|
||||
ragged_window: RaggedVerifyWindow,
|
||||
sampling_info,
|
||||
kv_loc_plan,
|
||||
) -> TargetVerifyResult:
|
||||
verify_input = DFlashVerifyInput(
|
||||
draft_token=ragged_window.verify_ids,
|
||||
@@ -429,6 +433,9 @@ class TargetVerifyExecutor:
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
ragged_verify_layout=layout,
|
||||
live_seq_lens_cpu=batch.seq_lens_cpu,
|
||||
# The packed rows are a selection of the planned verify window.
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
kv_loc_cols=ragged_window.window_index,
|
||||
)
|
||||
batch.out_cache_loc = ragged_window.verify_cache_loc
|
||||
seq_lens_cpu_backup = batch.seq_lens_cpu
|
||||
@@ -461,6 +468,7 @@ class TargetVerifyExecutor:
|
||||
bs: int,
|
||||
device: str,
|
||||
sampling_info,
|
||||
verify_window: VerifyWindow,
|
||||
inject_gate: bool = False,
|
||||
) -> tuple[TargetVerifyResult, torch.Tensor]:
|
||||
ragged_window = BuildRaggedVerifyWindow.execute(
|
||||
@@ -480,6 +488,7 @@ class TargetVerifyExecutor:
|
||||
layout=layout,
|
||||
ragged_window=ragged_window,
|
||||
sampling_info=sampling_info,
|
||||
kv_loc_plan=verify_window.kv_loc_plan,
|
||||
)
|
||||
logits_output = target_verify.logits_output
|
||||
|
||||
|
||||
@@ -642,6 +642,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
|
||||
target_hidden_is_projected = (
|
||||
@@ -705,7 +706,9 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
cache_loc = batch.out_cache_loc
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
|
||||
batch_output.kv_loc_plan = None
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
if token_indices is not None:
|
||||
cache_loc = cache_loc[token_indices]
|
||||
@@ -842,6 +845,11 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
block_pos_offsets=self._block_pos_offsets,
|
||||
model_runner=self.model_runner,
|
||||
seq_lens_cpu=(
|
||||
batch.seq_lens_cpu
|
||||
if batch.seq_lens_cpu is not None
|
||||
else draft_input.nxt_kv_lens_cpu
|
||||
),
|
||||
)
|
||||
|
||||
sampling_info = batch.sampling_info
|
||||
@@ -936,6 +944,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
bs=bs,
|
||||
device=device,
|
||||
sampling_info=sampling_info,
|
||||
verify_window=verify_window,
|
||||
inject_gate=fold_eligible,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -459,7 +459,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=num_seqs,
|
||||
input_ids=None,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -485,6 +484,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL
|
||||
),
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
def run_once():
|
||||
self.draft_attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
|
||||
@@ -424,7 +424,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -448,6 +447,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
if self.buffers.dsa_seed_topk_capture is not None:
|
||||
spec_info.dsa_seed_topk_capture = self.buffers.dsa_seed_topk_capture[
|
||||
@@ -649,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
encoder_lens=None,
|
||||
out_cache_loc=buffers.out_cache_loc[:num_tokens],
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
@@ -34,6 +35,9 @@ class EagleVerifyInput(SpecInput):
|
||||
draft_probs: torch.Tensor = None
|
||||
prepared_out_cache_loc: Optional[torch.Tensor] = None
|
||||
prepared_mrope_positions: Optional[torch.Tensor] = None
|
||||
# The iteration's plan when the draft planned the verify window
|
||||
# (`prepared_out_cache_loc`); verify and draft extend take it.
|
||||
kv_loc_plan: Optional[KVLocPlan] = None
|
||||
|
||||
# Shape info for padding
|
||||
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
|
||||
@@ -90,11 +94,12 @@ class EagleVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
device = req_pool_indices.device
|
||||
batch_size = req_pool_indices.numel()
|
||||
qo_indptr = torch.arange(
|
||||
@@ -113,13 +118,13 @@ class EagleVerifyInput(SpecInput):
|
||||
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
mask_numel = (
|
||||
paged_kernel_lens_sum * self.draft_token_num
|
||||
@@ -402,10 +407,11 @@ class EagleDraftExtendInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: Optional[int],
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""Draft-extend CSR args. The lens already include the window, so
|
||||
unlike verify nothing is widened here."""
|
||||
"""Draft-extend CSR args. ``paged_kernel_lens`` already includes the
|
||||
draft-extend window."""
|
||||
device = req_pool_indices.device
|
||||
bs = self.num_correct_drafts.numel()
|
||||
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
|
||||
@@ -426,12 +432,12 @@ class EagleDraftExtendInput(SpecInput):
|
||||
paged_kernel_lens_sum, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=paged_kernel_lens_sum,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
return kv_indices, cum_kv_seq_len, qo_indptr, None
|
||||
|
||||
@@ -607,6 +607,7 @@ def eagle_prepare_for_verify(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=verify_input.prepared_mrope_positions,
|
||||
kv_loc_plan=verify_input.kv_loc_plan,
|
||||
)
|
||||
|
||||
# Run attention backend plan and cuda graph preparation
|
||||
|
||||
@@ -55,6 +55,8 @@ if _is_cpu:
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
|
||||
@@ -121,7 +123,10 @@ def prepare_for_draft_extend(
|
||||
return_hidden_states_before_norm: bool,
|
||||
widened_out_cache_loc: Optional[torch.Tensor] = None,
|
||||
widened_positions: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""``kv_loc_plan``: the iteration's plan, whose window the draft extend
|
||||
writes again (the verify window); a widened window plans its own."""
|
||||
bs = len(batch.seq_lens)
|
||||
# Optional window widening (num_front_tokens=0 -> off): prepend that many
|
||||
# rows below the boundary. Locs/positions arrive precomputed; token/hidden
|
||||
@@ -204,6 +209,7 @@ def prepare_for_draft_extend(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=return_hidden_states_before_norm,
|
||||
extend_position_info=extend_position_info,
|
||||
kv_loc_plan=None if widen else kv_loc_plan,
|
||||
)
|
||||
# Forward sees post-write length (draft extend writes num_draft_tokens
|
||||
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
|
||||
@@ -243,18 +249,35 @@ def prepare_for_draft(
|
||||
draft_model_runner: ModelRunner,
|
||||
topk: int,
|
||||
num_steps: int,
|
||||
):
|
||||
|
||||
*,
|
||||
target_translator: Optional[KVIndexTranslator] = None,
|
||||
num_draft_tokens: Optional[int] = None,
|
||||
) -> tuple[ForwardBatch, bool, Optional[KVLocPlan]]:
|
||||
"""The draft-decode batch. With ``target_translator``, a chain draft plans
|
||||
the iteration's whole write window (the ``num_draft_tokens`` slots past
|
||||
``seq_lens`` that verify and draft extend also write) and writes its first
|
||||
``num_steps`` columns; that plan is returned, else None."""
|
||||
prepared_positions = None
|
||||
prepared_mrope_positions = None
|
||||
kv_loc_plan = None
|
||||
if not batch.forward_mode.is_idle():
|
||||
bs = len(batch.seq_lens)
|
||||
|
||||
# Assign cache locations (draft-write targets).
|
||||
page_size = batch.token_to_kv_pool_allocator.page_size
|
||||
plans_window = (
|
||||
target_translator is not None
|
||||
and topk == 1
|
||||
and num_draft_tokens is not None
|
||||
and num_draft_tokens > num_steps
|
||||
and not _is_cpu
|
||||
)
|
||||
# A chain draft's slots lead the iteration's window: gather the whole
|
||||
# window in the same launch.
|
||||
window_cols = num_draft_tokens if plans_window else num_steps
|
||||
if page_size == 1 or topk == 1:
|
||||
batch.out_cache_loc = torch.empty(
|
||||
(bs * topk * num_steps,),
|
||||
(bs * topk * window_cols,),
|
||||
dtype=torch.int64,
|
||||
device=batch.device,
|
||||
)
|
||||
@@ -285,13 +308,21 @@ def prepare_for_draft(
|
||||
batch.out_cache_loc,
|
||||
req_to_token_pool.req_to_token.shape[1],
|
||||
topk,
|
||||
num_steps,
|
||||
window_cols,
|
||||
positions=prepared_positions,
|
||||
mrope=prepared_mrope_positions,
|
||||
BS=bs,
|
||||
WRITE_POSITIONS=prepared_positions is not None,
|
||||
WRITE_MROPE=prepared_mrope_positions is not None,
|
||||
)
|
||||
if plans_window:
|
||||
kv_loc_plan = target_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=batch.seq_lens,
|
||||
seq_lens_cpu=batch.seq_lens_cpu,
|
||||
write_virtual=batch.out_cache_loc,
|
||||
read_extent=num_draft_tokens,
|
||||
)
|
||||
else:
|
||||
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
|
||||
# Reduce out_cache_loc from the page-aligned tree region down to the
|
||||
@@ -352,11 +383,13 @@ def prepare_for_draft(
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=prepared_mrope_positions,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
write_cols=None if kv_loc_plan is None else slice(0, num_steps),
|
||||
)
|
||||
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
|
||||
forward_batch
|
||||
)
|
||||
return forward_batch, can_run_decode_cuda_graph
|
||||
return forward_batch, can_run_decode_cuda_graph, kv_loc_plan
|
||||
|
||||
|
||||
def build_eagle_verify_input(
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
@@ -606,7 +607,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
def draft(self, batch: ScheduleBatch, *, with_topology: bool = False):
|
||||
draft_input: EagleDraftInput = batch.spec_info
|
||||
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
|
||||
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
|
||||
draft_input,
|
||||
self.req_to_token_pool,
|
||||
batch,
|
||||
@@ -614,6 +615,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
target_translator=self.target_worker.model_runner.kv_index_translator,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
)
|
||||
if (
|
||||
can_run_decode_cuda_graph
|
||||
@@ -673,6 +676,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
tree_mask_mode=self.tree_mask_mode,
|
||||
device=self.device,
|
||||
)
|
||||
if kv_loc_plan is not None:
|
||||
# Verify and draft extend write the window the draft planned.
|
||||
verify_input.kv_loc_plan = kv_loc_plan
|
||||
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
|
||||
if with_topology:
|
||||
# PP+spec relays the tree so every stage rebuilds the same verify
|
||||
# input; the mask build needs the topology this one was built from.
|
||||
@@ -900,6 +907,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
target_hidden_states: torch.Tensor,
|
||||
next_token_ids: torch.Tensor,
|
||||
mm_input_embeds: Optional[torch.Tensor] = None,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""
|
||||
Run draft model extend to correctly fill the KV cache.
|
||||
@@ -908,6 +916,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: The batch to run.
|
||||
target_hidden_states: Hidden states from the target model forward
|
||||
next_token_ids: Next token ids generated from the target forward.
|
||||
kv_loc_plan: The target prefill's plan; the draft writes the same
|
||||
slots.
|
||||
"""
|
||||
# Construct input_ids
|
||||
if not batch.forward_mode.is_idle():
|
||||
@@ -959,6 +969,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
forward_batch.return_logprob = False
|
||||
if mm_input_embeds is not None:
|
||||
@@ -1035,7 +1046,11 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
return buf[:num_tokens]
|
||||
|
||||
def _draft_extend_for_decode(
|
||||
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
batch_result: GenerationBatchResult,
|
||||
*,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
# Cast to int64 before entering plan stream to avoid cross-stream
|
||||
# synchronization issues with .to() inside the plan stream context.
|
||||
@@ -1093,6 +1108,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner,
|
||||
self.cuda_graph_runner_for_draft_extend,
|
||||
return_hidden_states_before_norm=False,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.plan_stream:
|
||||
@@ -1412,7 +1428,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
):
|
||||
self.draft_worker._draft_extend_for_decode(batch, batch_output)
|
||||
self.draft_worker._draft_extend_for_decode(
|
||||
batch, batch_output, kv_loc_plan=verify_input.kv_loc_plan
|
||||
)
|
||||
|
||||
if (
|
||||
get_parallel().pp_size > 1
|
||||
@@ -1468,7 +1486,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
return_kv_loc_plan=self._draft_worker is not None,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
@@ -1505,6 +1525,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
batch_output.logits_output.mm_input_embeds,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
@@ -302,7 +302,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=expanded_bs,
|
||||
input_ids=None,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -321,6 +320,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
|
||||
def run_once():
|
||||
# Record the metadata rebuild against the committed target-prefix
|
||||
|
||||
@@ -365,6 +365,8 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
encoder_lens=None,
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan,
|
||||
kv_loc_cols=forward_batch.kv_loc_cols,
|
||||
spec_info=None,
|
||||
)
|
||||
with self._frozen_kv_target_view(forward_batch):
|
||||
|
||||
@@ -99,6 +99,7 @@ if is_npu():
|
||||
fill_draft_extend_prepare_buffers = fill_draft_extend_prepare_buffers_native
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.speculative.multi_layer_eagle_worker_v2 import (
|
||||
MultiLayerEagleDraftWorker,
|
||||
)
|
||||
@@ -333,7 +334,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Forward batch
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=self.forward_mode,
|
||||
out_cache_loc_is_physical=True,
|
||||
batch_size=bs,
|
||||
input_ids=input_ids,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -361,6 +361,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
global_num_token_non_padded_cpu=self.captured_req_width * bs,
|
||||
return_hidden_states_before_norm=True,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
return forward_batch
|
||||
|
||||
def _postprocess_forward_batch(self, forward_batch: ForwardBatch, bs: int):
|
||||
@@ -476,6 +477,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
*,
|
||||
out_cache_loc_virtual: Optional[torch.Tensor],
|
||||
kv_loc_plan: Optional[KVLocPlan],
|
||||
kv_loc_cols: Optional[Cols],
|
||||
):
|
||||
"""Init this step's attention metadata for the prepared bucket and
|
||||
replay its graph. Buffers must already be populated by the composite
|
||||
@@ -501,6 +504,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Virtual input stays separate from the backend's physical buffer.
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
out_cache_loc_virtual=out_cache_loc_virtual,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
kv_loc_cols=kv_loc_cols,
|
||||
spec_info=spec_info,
|
||||
)
|
||||
if (
|
||||
@@ -550,6 +555,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
self.num_front_tokens = 0
|
||||
self.prune_draft_extend_logits = False
|
||||
self._out_cache_loc_virtual = None
|
||||
self._kv_loc_plan = None
|
||||
self._kv_loc_cols = None
|
||||
|
||||
self._init_and_capture()
|
||||
|
||||
@@ -740,11 +747,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
req_pool_indices,
|
||||
out_cache_loc,
|
||||
positions=None,
|
||||
out_cache_loc_virtual=None,
|
||||
out_cache_loc_virtual,
|
||||
kv_loc_plan,
|
||||
):
|
||||
# Staging runs before `prepare`, so the rail arrives here rather than
|
||||
# off a ForwardBatch; `prepare` overwrites it with the batch's own.
|
||||
# Staging runs before `prepare` has a ForwardBatch, so the virtual write
|
||||
# ids and the plan come in here; `prepare` replaces them with the batch's.
|
||||
self._out_cache_loc_virtual = out_cache_loc_virtual
|
||||
self._kv_loc_plan = kv_loc_plan
|
||||
self._kv_loc_cols = None
|
||||
raw_bs = req_pool_indices.shape[0]
|
||||
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
|
||||
buffers = self.buffers
|
||||
@@ -780,6 +790,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
|
||||
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
|
||||
out_cache_loc_virtual=self._out_cache_loc_virtual,
|
||||
kv_loc_plan=self._kv_loc_plan,
|
||||
kv_loc_cols=self._kv_loc_cols,
|
||||
)
|
||||
for backend in backends:
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
@@ -792,6 +804,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
num_tokens = raw_bs * self.captured_req_width
|
||||
|
||||
self._out_cache_loc_virtual = forward_batch.out_cache_loc_virtual
|
||||
self._kv_loc_plan = forward_batch.kv_loc_plan
|
||||
self._kv_loc_cols = forward_batch.kv_loc_cols
|
||||
|
||||
# Bucketize to a captured batch size (padding the tail).
|
||||
if self.require_mlp_tp_gather:
|
||||
@@ -900,6 +914,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
self._replay_spec_info,
|
||||
self.seq_lens_cpu,
|
||||
out_cache_loc_virtual=self._out_cache_loc_virtual,
|
||||
kv_loc_plan=self._kv_loc_plan,
|
||||
kv_loc_cols=self._kv_loc_cols,
|
||||
)
|
||||
raw_bs = self.raw_bs
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
|
||||
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, List, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -107,6 +107,7 @@ _is_cpu = is_cpu()
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner, ModelRunnerOutput
|
||||
|
||||
|
||||
@@ -460,7 +461,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
def draft(self, batch: ScheduleBatch):
|
||||
draft_input: EagleDraftInput = batch.spec_info
|
||||
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
|
||||
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
|
||||
draft_input,
|
||||
self.req_to_token_pool,
|
||||
batch,
|
||||
@@ -468,12 +469,14 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner_list[0],
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
target_translator=self.target_worker.model_runner.kv_index_translator,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
)
|
||||
|
||||
# Run draft
|
||||
parent_list, top_scores_index, draft_tokens = self.draft_forward(forward_batch)
|
||||
|
||||
return build_eagle_verify_input(
|
||||
verify_input = build_eagle_verify_input(
|
||||
batch,
|
||||
draft_input,
|
||||
parent_list,
|
||||
@@ -487,6 +490,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
tree_mask_mode=self.tree_mask_mode,
|
||||
device=self.device,
|
||||
)
|
||||
if kv_loc_plan is not None:
|
||||
# Verify and draft extend write the window the draft planned.
|
||||
verify_input.kv_loc_plan = kv_loc_plan
|
||||
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
|
||||
return verify_input
|
||||
|
||||
def draft_forward(self, forward_batch: ForwardBatch):
|
||||
# Parse args
|
||||
@@ -596,6 +604,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: ScheduleBatch,
|
||||
target_hidden_states: torch.Tensor,
|
||||
next_token_ids: torch.Tensor,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
"""
|
||||
Run draft model extend to correctly fill the KV cache.
|
||||
@@ -604,6 +613,8 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch: The batch to run.
|
||||
target_hidden_states: Hidden states from the target model forward
|
||||
next_token_ids: Next token ids generated from the target forward.
|
||||
kv_loc_plan: The target prefill's plan; the draft writes the same
|
||||
slots.
|
||||
"""
|
||||
# The draft embed clamps unconditionally (to tolerate multimodal pad
|
||||
# sentinels), so probe next_token_ids here first -- otherwise a corrupted id
|
||||
@@ -641,6 +652,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner_list[0],
|
||||
capture_hidden_mode=draft_capture_hidden_mode,
|
||||
return_hidden_states_before_norm=True,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
self._apply_deferred_mamba_init_to_draft_pools(forward_batch)
|
||||
@@ -766,14 +778,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
draft_token_num=self.speculative_num_draft_tokens,
|
||||
device=batch.device,
|
||||
)
|
||||
seq_lens = batch.seq_lens + self.speculative_num_draft_tokens
|
||||
runner.stage_shared_reads(
|
||||
seq_lens=batch.seq_lens + self.speculative_num_draft_tokens,
|
||||
seq_lens=seq_lens,
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
out_cache_loc=locs,
|
||||
positions=positions,
|
||||
# Both loc sources read req_to_token untranslated, so `locs` is the
|
||||
# PRE-translate write loc the capture rail needs.
|
||||
# Both loc sources read req_to_token untranslated: `locs` is virtual.
|
||||
out_cache_loc_virtual=locs,
|
||||
kv_loc_plan=self.draft_runner_list[0].kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=None,
|
||||
write_virtual=locs,
|
||||
),
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -783,6 +801,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
batch_result: GenerationBatchResult,
|
||||
*,
|
||||
staged: bool = False,
|
||||
kv_loc_plan: Optional[KVLocPlan] = None,
|
||||
):
|
||||
# Batch 2: Draft extend
|
||||
draft_extend_input = EagleDraftExtendInput(
|
||||
@@ -826,6 +845,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
return_hidden_states_before_norm=True,
|
||||
widened_out_cache_loc=boundary_kv_locs,
|
||||
widened_positions=boundary_kv_positions,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
|
||||
if self.plan_stream:
|
||||
@@ -1091,7 +1111,9 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
@@ -1106,6 +1128,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
batch,
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
kv_loc_plan=kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
else:
|
||||
@@ -1134,7 +1157,10 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
if on_publish is not None:
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
self.draft_worker._draft_extend_for_decode(
|
||||
batch, batch_output, staged=staged
|
||||
batch,
|
||||
batch_output,
|
||||
staged=staged,
|
||||
kv_loc_plan=verify_input.kv_loc_plan,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
|
||||
|
||||
@@ -63,11 +64,12 @@ class NgramVerifyInput(SpecInput):
|
||||
paged_kernel_lens: torch.Tensor,
|
||||
paged_kernel_lens_sum: int,
|
||||
translator: KVIndexTranslator,
|
||||
sliding_window: bool = False,
|
||||
plan: KVLocPlan,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
):
|
||||
"""CSR verify args, gathered straight into the packed stream. The lens
|
||||
are widened here and handed to the translator, so nothing materializes
|
||||
a ``[bs, max_pages]`` rectangle to repack from."""
|
||||
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
|
||||
is widened here; the translator packs the read ids straight into
|
||||
``kv_indices``."""
|
||||
bs = req_pool_indices.numel()
|
||||
|
||||
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
|
||||
@@ -83,13 +85,13 @@ class NgramVerifyInput(SpecInput):
|
||||
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
|
||||
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
|
||||
|
||||
translator.fill_packed_read_stream(
|
||||
translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=paged_kernel_lens,
|
||||
indptr=cum_kv_seq_len,
|
||||
total_tokens=total_tokens,
|
||||
out=kv_indices,
|
||||
sliding_window=sliding_window,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.
|
||||
|
||||
@@ -297,7 +297,7 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
draft_token_num=self.forward_width,
|
||||
)
|
||||
|
||||
return ForwardBatch(
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.TARGET_VERIFY,
|
||||
batch_size=len(prefix_lens),
|
||||
input_ids=input_ids,
|
||||
@@ -312,6 +312,8 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_own_plan(forward_batch)
|
||||
return forward_batch
|
||||
|
||||
def _run_target_block(
|
||||
self,
|
||||
|
||||
@@ -761,6 +761,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
batch.extend_prefix_lens = torch.tensor(
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
@@ -419,6 +420,13 @@ class MockDSV4ModelRunner:
|
||||
identity = torch.arange(swa_size, dtype=torch.int64, device=device)
|
||||
self.token_to_kv_pool.register_mapping(identity)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.kv_index_translator = KVIndexTranslator(
|
||||
req_to_token=self.req_to_token_pool.req_to_token,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
page_size=case.page_size,
|
||||
device=device,
|
||||
)
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
self.init_new_workspace = False
|
||||
@@ -661,6 +669,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
# extend_* fields are only populated for extend-shaped modes. DECODE leaves
|
||||
# them at their defaults; the flash_mla path reads metadata directly from
|
||||
# DSV4AttnMetadata so the extend fields are unused for the compress_ratio=0
|
||||
|
||||
@@ -322,6 +322,7 @@ class MockGDNModelRunner(ModelRunner):
|
||||
page_size=case.page_size,
|
||||
get_kvcache=lambda: self.token_to_kv_pool,
|
||||
)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -329,7 +330,6 @@ class MockGDNModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -562,6 +562,8 @@ def _make_forward_batch(
|
||||
else None
|
||||
),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -318,6 +318,7 @@ class MockKDAModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -325,7 +326,6 @@ class MockKDAModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -552,6 +552,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -327,6 +327,7 @@ class MockLightningModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -334,7 +335,6 @@ class MockLightningModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -523,6 +523,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -460,7 +461,6 @@ class MockMamba2ModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = False
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
|
||||
@property
|
||||
@@ -653,6 +653,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -307,6 +307,7 @@ class MockMLAModelRunner(ModelRunner):
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.is_draft_worker = False
|
||||
self.init_kv_index_translator()
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
@@ -314,7 +315,6 @@ class MockMLAModelRunner(ModelRunner):
|
||||
self.is_hybrid_swa = False
|
||||
self.sliding_window_size = None
|
||||
self.use_mla_backend = True
|
||||
self.is_draft_worker = False
|
||||
self._kernel_warmed_up = True
|
||||
# Runner-mode helpers mutate speculative graph sizes after construction.
|
||||
self.graph_shared_output = GraphSharedOutput(
|
||||
@@ -693,6 +693,8 @@ def _make_forward_batch(
|
||||
seq_lens_sum=sum(seq_lens),
|
||||
positions=torch.tensor(positions, dtype=torch.int64, device=device),
|
||||
)
|
||||
# Production batches take their KV ids from a plan (`init_new`).
|
||||
runner.kv_index_translator.bind_own_plan(batch)
|
||||
|
||||
if case.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -297,6 +297,8 @@ def _init_cuda_graph_replay_metadata(backend, capture_batch_size: int, batch):
|
||||
encoder_lens=batch.encoder_lens,
|
||||
extend_seq_lens=getattr(batch, "extend_seq_lens", None),
|
||||
out_cache_loc=getattr(batch, "out_cache_loc", None),
|
||||
kv_loc_plan=batch.kv_loc_plan,
|
||||
kv_loc_cols=batch.kv_loc_cols,
|
||||
spec_info=batch.spec_info,
|
||||
)
|
||||
backend.init_forward_metadata_out_graph(fb_view)
|
||||
|
||||
@@ -510,6 +510,7 @@ def run_eagle_draft_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
|
||||
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
|
||||
expected = _run_eagle_draft_eager(
|
||||
eager_worker,
|
||||
eager_batch,
|
||||
@@ -526,6 +527,7 @@ def run_eagle_draft_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
|
||||
graph_runner = _capture_eagle_draft_graph_runner(
|
||||
graph_worker,
|
||||
graph_backend,
|
||||
@@ -565,6 +567,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
|
||||
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
|
||||
expected = _run_frozen_kv_mtp_eager(eager_worker, eager_batch)
|
||||
|
||||
graph_fixture, graph_worker, _ = _build_frozen_kv_mtp_fixture(
|
||||
@@ -576,6 +579,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
|
||||
)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
|
||||
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
|
||||
graph_runner = _capture_frozen_kv_mtp_graph_runner(graph_worker)
|
||||
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ FULL side's ids while it used to be sized by its own page count. A model with
|
||||
few full-attention layers and many sliding ones gives the id owner more pages,
|
||||
and the top of that id space then lands off the end of the swa table. On GPU
|
||||
the write is unchecked (`alloc_bind_inplace`'s `tl.store`), so the symptom
|
||||
surfaces later and elsewhere: a device-side index assert on the read in
|
||||
`_swa_write_loc_unified`, and a dead scheduler.
|
||||
surfaces later and elsewhere: a device-side index assert on a later read of
|
||||
the swa table, and a dead scheduler.
|
||||
|
||||
The reachability of it is a KV-budget property, not just a model property: ids
|
||||
come off the front of the owner's free list and freed ids return to the back, so
|
||||
|
||||
@@ -55,6 +55,7 @@ def _apply(backend, *, bs: int, seq_lens: torch.Tensor, forward_mode):
|
||||
patch.object(backend, "_fill_dcp_block_kv_indices") as fill,
|
||||
):
|
||||
backend._apply_cuda_graph_metadata(
|
||||
plan=None, # DCP builds its own block table
|
||||
bs=bs,
|
||||
req_pool_indices=torch.arange(bs, dtype=torch.int32, device="cuda"),
|
||||
seq_lens=seq_lens,
|
||||
@@ -108,7 +109,6 @@ class _DCPMetadataTests:
|
||||
heads, v_head_dim, n = 4, 512, 3 * NUM_DRAFT_TOKENS
|
||||
backend = object.__new__(self.backend_cls)
|
||||
backend.data_type = backend.q_data_type = torch.bfloat16
|
||||
backend._decode_kernel_loc = None
|
||||
|
||||
def real_path(*args, **kwargs):
|
||||
raise _RealVerifyPath
|
||||
|
||||
@@ -24,7 +24,7 @@ class TestKimiLinearUnifiedMemoryFlashMLA(
|
||||
GSM8KMixin, PrefixCacheBranchingMixin, DefaultServerBase
|
||||
):
|
||||
"""flashmla at its ps=64 snap: the block-table route
|
||||
(KVIndexTranslator.fill_read_table into flashmla's padded tables) plus the
|
||||
(the plan's page table copied into flashmla's padded tables) plus the
|
||||
ps=64 sub-pool sizing (64-token sink floor, per-layer-view tail pad).
|
||||
Hopper-only, like the rest of this nightly suite."""
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.layers.attention.linear.inkling_sconv_backend import (
|
||||
InklingShortConvHybridAttnBackend,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
MambaSubPoolSpec,
|
||||
@@ -461,6 +462,9 @@ class TestInklingCheckpointIndices(CustomTestCase):
|
||||
composite.buffers = buffers
|
||||
composite.captured_req_width = 4
|
||||
composite.draft_extend_attn_backend_list = [wrapper]
|
||||
composite._out_cache_loc_virtual = None
|
||||
composite._kv_loc_plan = None
|
||||
composite._kv_loc_cols = None
|
||||
for step, tracking in enumerate((True, False, True)):
|
||||
with self.subTest(tracking=tracking, step=step):
|
||||
ids = slots[-4:].roll(step).clone()
|
||||
@@ -814,6 +818,15 @@ class TestInklingCheckpointIndices(CustomTestCase):
|
||||
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
|
||||
device_timer=None,
|
||||
canary_manager=None,
|
||||
# The KV side is not under test: a static pool's translator binds
|
||||
# the captured batch to its own write ids.
|
||||
kv_index_translator=KVIndexTranslator(
|
||||
req_to_token=torch.zeros((1, 8), dtype=torch.int32, device="cuda"),
|
||||
token_to_kv_pool_allocator=None,
|
||||
token_to_kv_pool=object(),
|
||||
page_size=1,
|
||||
device="cuda",
|
||||
),
|
||||
)
|
||||
return runner, backend, allocator, pool, slots, buffers
|
||||
|
||||
@@ -926,6 +939,9 @@ class TestInklingCheckpointIndices(CustomTestCase):
|
||||
input_ids=buffers.input_ids[: live * 4],
|
||||
positions=buffers.positions[: live * 4],
|
||||
out_cache_loc=buffers.out_cache_loc[: live * 4],
|
||||
out_cache_loc_virtual=None,
|
||||
kv_loc_plan=None,
|
||||
kv_loc_cols=None,
|
||||
req_pool_indices=torch.arange(1, live + 1, device="cuda"),
|
||||
seq_lens=torch.full((live,), 6, dtype=torch.int64, device="cuda"),
|
||||
seq_lens_sum=6 * live,
|
||||
@@ -973,7 +989,13 @@ class TestInklingCheckpointIndices(CustomTestCase):
|
||||
composite._stage_metadata = Mock()
|
||||
composite.prepare(fresh)
|
||||
runner.replay(
|
||||
4, composite.seq_lens_sum, composite._replay_spec_info, None
|
||||
4,
|
||||
composite.seq_lens_sum,
|
||||
composite._replay_spec_info,
|
||||
None,
|
||||
out_cache_loc_virtual=composite._out_cache_loc_virtual,
|
||||
kv_loc_plan=composite._kv_loc_plan,
|
||||
kv_loc_cols=composite._kv_loc_cols,
|
||||
)
|
||||
else:
|
||||
runner.execute(fresh, torch.arange(live, device="cuda") * 4 + 1)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`generate_draft_decode_kv_indices` over the unified pool's read tables.
|
||||
|
||||
Over the plan's physical page table (token ids rebuilt as
|
||||
`entry * ps + pos % ps`), and over virtual rows translated in the kernel's own
|
||||
gather, the kernel must emit exactly the translated virtual ids and the same
|
||||
kv_indptr. The static `req_to_token` path is covered by
|
||||
test_spec_kv_indices_grid.py.
|
||||
|
||||
python -m pytest test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
_SENTINEL = -7
|
||||
|
||||
|
||||
def _run_kernel(
|
||||
*, table, entry_page_size, seq_lens, positions, num_steps, topk, page_size, v2p=None
|
||||
):
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_kv_indices,
|
||||
)
|
||||
from sglang.srt.utils import next_power_of_2
|
||||
|
||||
num_seqs = seq_lens.numel()
|
||||
bs = num_seqs * topk
|
||||
width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64)
|
||||
kv_indices = torch.full(
|
||||
(num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda")
|
||||
generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)](
|
||||
torch.arange(num_seqs, dtype=torch.int64, device="cuda"),
|
||||
table,
|
||||
seq_lens,
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
positions,
|
||||
table.stride(0),
|
||||
width,
|
||||
kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
next_power_of_2(num_steps),
|
||||
next_power_of_2(bs),
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE=entry_page_size,
|
||||
v2p=v2p,
|
||||
TRANSLATE=v2p is not None,
|
||||
)
|
||||
return kv_indices, kv_indptr
|
||||
|
||||
|
||||
class TestDraftDecodeKVIndicesReadTable(CustomTestCase):
|
||||
def _check(self, *, page_size, num_steps=3, topk=1):
|
||||
torch.manual_seed(7)
|
||||
num_seqs, max_context = 3, 64
|
||||
num_pages = max_context // page_size
|
||||
# Each row holds whole virtual pages in a shuffled order, and the
|
||||
# virtual->physical table is a shuffle too, so nothing is identity.
|
||||
v2p = torch.randperm(num_pages, device="cuda")
|
||||
rows = []
|
||||
for _ in range(num_seqs):
|
||||
vpages = torch.randperm(num_pages, device="cuda")
|
||||
offsets = torch.arange(page_size, device="cuda")
|
||||
rows.append((vpages[:, None] * page_size + offsets).reshape(-1))
|
||||
req_to_token = torch.stack(rows)
|
||||
# The plan's table: one physical page per virtual page of the row.
|
||||
table = v2p[req_to_token[:, ::page_size] // page_size].to(torch.int32)
|
||||
seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")
|
||||
positions = seq_lens.repeat_interleave(topk)
|
||||
common = dict(
|
||||
seq_lens=seq_lens,
|
||||
positions=positions,
|
||||
num_steps=num_steps,
|
||||
topk=topk,
|
||||
page_size=page_size,
|
||||
)
|
||||
raw, raw_indptr = _run_kernel(table=req_to_token, entry_page_size=1, **common)
|
||||
out, out_indptr = _run_kernel(table=table, entry_page_size=page_size, **common)
|
||||
# No shared table: the virtual rows, translated in the kernel's gather.
|
||||
gathered, gathered_indptr = _run_kernel(
|
||||
table=req_to_token, entry_page_size=1, v2p=v2p, **common
|
||||
)
|
||||
|
||||
torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0)
|
||||
written = raw != _SENTINEL
|
||||
self.assertTrue(bool((out[~written] == _SENTINEL).all()))
|
||||
virt = raw[written]
|
||||
expected = v2p[virt // page_size] * page_size + virt % page_size
|
||||
torch.testing.assert_close(out[written], expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(gathered_indptr, out_indptr, rtol=0, atol=0)
|
||||
torch.testing.assert_close(gathered, out, rtol=0, atol=0)
|
||||
|
||||
def test_page_size_one(self):
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
self._check(page_size=1)
|
||||
|
||||
def test_page_size_two(self):
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
self._check(page_size=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -117,7 +117,6 @@ def _run(io, window_size=0, sink_size=0):
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
io["positions"],
|
||||
None, # v2p: no unified-pool translation
|
||||
io["pool_len"],
|
||||
kv_indices.shape[1],
|
||||
kv_indptr.shape[1],
|
||||
|
||||
-172
@@ -1,172 +0,0 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`generate_draft_decode_kv_indices` under the unified pool's translation.
|
||||
|
||||
BUG REGRESSION. The multi-step draft-decode kernel copied req_to_token values
|
||||
verbatim into the per-step kv_indices. Under `--enable-unified-memory` those
|
||||
values are VIRTUAL ids while the fused draft pool is indexed by physical
|
||||
ids, so with `speculative_num_steps > 1` every draft decode step read the
|
||||
wrong KV rows -- silently: bad drafts only depress acceptance, verify stays
|
||||
correct. Pinned here: TRANSLATE=True must emit exactly `translate_kv_loc`'s
|
||||
physical ids, and TRANSLATE=False must stay byte-identical to the historical
|
||||
raw copy.
|
||||
|
||||
python -m pytest test/registered/kernels/test_generate_draft_decode_kv_indices_translate.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
_SENTINEL = -7
|
||||
|
||||
|
||||
def _run_kernel(
|
||||
*,
|
||||
req_to_token,
|
||||
seq_lens,
|
||||
positions,
|
||||
num_steps,
|
||||
topk,
|
||||
page_size,
|
||||
v2p,
|
||||
translate,
|
||||
):
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_kv_indices,
|
||||
)
|
||||
from sglang.srt.utils import next_power_of_2
|
||||
|
||||
num_seqs = seq_lens.numel()
|
||||
bs = num_seqs * topk
|
||||
width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64)
|
||||
kv_indices = torch.full(
|
||||
(num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda")
|
||||
generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)](
|
||||
torch.arange(num_seqs, dtype=torch.int64, device="cuda"),
|
||||
req_to_token,
|
||||
seq_lens,
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
positions,
|
||||
v2p if translate else None,
|
||||
req_to_token.shape[1],
|
||||
width,
|
||||
kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
next_power_of_2(num_steps),
|
||||
next_power_of_2(bs),
|
||||
page_size,
|
||||
TRANSLATE=translate,
|
||||
)
|
||||
return kv_indices, kv_indptr
|
||||
|
||||
|
||||
class TestGenerateDraftDecodeKVIndicesTranslate(CustomTestCase):
|
||||
def _check(self, *, page_size, num_steps=3, topk=1, unused_slots=False):
|
||||
torch.manual_seed(7)
|
||||
num_seqs, max_context = 3, 64
|
||||
num_pages = max_context // page_size + 1
|
||||
# Virtual ids: a shuffled page layout so translation is not identity.
|
||||
perm = torch.randperm(num_pages, device="cuda")
|
||||
v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda")
|
||||
v2p[perm] = torch.arange(num_pages, device="cuda")
|
||||
req_to_token = (
|
||||
torch.arange(max_context, dtype=torch.int64, device="cuda")
|
||||
.flip(0)
|
||||
.repeat(num_seqs, 1)
|
||||
)
|
||||
if unused_slots:
|
||||
# Negative entries inside the read window must land on the sink.
|
||||
req_to_token[:, ::3] = -1
|
||||
seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")[:num_seqs]
|
||||
positions = seq_lens.repeat_interleave(topk)
|
||||
|
||||
common = dict(
|
||||
req_to_token=req_to_token,
|
||||
seq_lens=seq_lens,
|
||||
positions=positions,
|
||||
num_steps=num_steps,
|
||||
topk=topk,
|
||||
page_size=page_size,
|
||||
v2p=v2p,
|
||||
)
|
||||
raw, raw_indptr = _run_kernel(translate=False, **common)
|
||||
out, out_indptr = _run_kernel(translate=True, **common)
|
||||
|
||||
torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0)
|
||||
written = raw != _SENTINEL
|
||||
# Unwritten lanes stay untouched in both compilations.
|
||||
self.assertTrue(bool((out[~written] == _SENTINEL).all()))
|
||||
# Written lanes: translate_kv_loc's formula over the raw ids.
|
||||
virt = raw[written]
|
||||
page = torch.where(virt < 0, 0, virt // page_size)
|
||||
expected = torch.where(
|
||||
virt < 0,
|
||||
0,
|
||||
torch.clamp_min(v2p[page] * page_size + virt % page_size, 0),
|
||||
)
|
||||
torch.testing.assert_close(out[written], expected, rtol=0, atol=0)
|
||||
|
||||
def test_page_size_one(self):
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
self._check(page_size=1)
|
||||
|
||||
def test_page_size_two(self):
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
self._check(page_size=2)
|
||||
|
||||
def test_negative_ids_map_to_the_sink(self):
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
for page_size in (1, 2):
|
||||
self._check(page_size=page_size, unused_slots=True)
|
||||
|
||||
def test_translate_false_is_a_raw_copy(self):
|
||||
"""The static-pool compilation must stay byte-identical to the
|
||||
historical kernel: the emitted indices ARE the req_to_token values."""
|
||||
if not torch.cuda.is_available():
|
||||
self.skipTest("CUDA required")
|
||||
torch.manual_seed(11)
|
||||
num_seqs, max_context, page_size = 2, 32, 1
|
||||
req_to_token = torch.randint(
|
||||
0, 1 << 20, (num_seqs, max_context), dtype=torch.int64, device="cuda"
|
||||
)
|
||||
seq_lens = torch.tensor([4, 7], dtype=torch.int64, device="cuda")
|
||||
raw, _ = _run_kernel(
|
||||
req_to_token=req_to_token,
|
||||
seq_lens=seq_lens,
|
||||
positions=seq_lens,
|
||||
num_steps=2,
|
||||
topk=1,
|
||||
page_size=page_size,
|
||||
v2p=None,
|
||||
translate=False,
|
||||
)
|
||||
written = raw != _SENTINEL
|
||||
emitted = raw[written]
|
||||
self.assertTrue(bool(torch.isin(emitted, req_to_token.flatten()).all()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -60,7 +60,6 @@ def _run_draft(kern_inputs, topk, steps, page_size, nb, kw):
|
||||
kv_i,
|
||||
kv_p,
|
||||
positions,
|
||||
None, # v2p: no unified-pool translation
|
||||
POOL_LEN,
|
||||
kv_i.shape[1],
|
||||
kv_p.shape[1],
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise
|
||||
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
|
||||
DecodeCudaGraphRunner,
|
||||
@@ -26,16 +27,16 @@ class _ReadStream:
|
||||
def __init__(self):
|
||||
self.tokens = torch.arange(64).view(4, 16)
|
||||
|
||||
def fill_packed_read_stream(
|
||||
def pack_read_stream(
|
||||
self,
|
||||
plan,
|
||||
*,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
indptr,
|
||||
total_tokens,
|
||||
out,
|
||||
kv_start_idx=None,
|
||||
sliding_window=False,
|
||||
kind=IdSpaceKind.FULL,
|
||||
token_mapping=None,
|
||||
):
|
||||
for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)):
|
||||
@@ -91,6 +92,7 @@ class TestGemma4GraphMetadata(unittest.TestCase):
|
||||
requests = torch.tensor([1, 2, 0])
|
||||
# Prefixes 3, 11, and an empty padded request; each has a four-token canvas.
|
||||
backend._apply_cuda_graph_metadata(
|
||||
plan=None,
|
||||
bs=3,
|
||||
req_pool_indices=requests,
|
||||
seq_lens=torch.tensor([7, 15, 4]),
|
||||
@@ -126,6 +128,7 @@ class TestGemma4GraphMetadata(unittest.TestCase):
|
||||
metadata.window_kv_indices.data_ptr(),
|
||||
)
|
||||
backend._apply_cuda_graph_metadata(
|
||||
plan=None,
|
||||
bs=3,
|
||||
req_pool_indices=torch.tensor([3, 1, 0]),
|
||||
seq_lens=torch.tensor([4, 9, 4]),
|
||||
|
||||
@@ -89,10 +89,8 @@ class TestFlashAttentionGraphMetadata(CustomTestCase):
|
||||
|
||||
|
||||
class TestSpecReadSeqLenDelta(CustomTestCase):
|
||||
"""The mode -> seq_len_delta map IS the widening the whole-sequence spec
|
||||
reads apply to cache_seqlens; a drift silently truncates the verify or
|
||||
draft tail of the translated read table (the metadata builders widen
|
||||
cache_seqlens while the source fill stays prefix-only)."""
|
||||
"""How far past seq_lens each spec mode's translated read table reaches;
|
||||
too short silently leaves the verify or draft tail stale."""
|
||||
|
||||
def _backend(self, *, topk=1, num_steps=3, step_id=1, num_draft=4):
|
||||
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
|
||||
@@ -127,11 +125,8 @@ class TestSpecReadSeqLenDelta(CustomTestCase):
|
||||
|
||||
|
||||
class TestDraftExtendInGraph(CustomTestCase):
|
||||
"""The draft-extend cuda-graph runners record
|
||||
`init_forward_metadata_in_graph` unconditionally. Under translation fa3
|
||||
rebuilds the draft-extend tables out of graph on every replay, so the
|
||||
in-graph gather -- raw req_to_token, i.e. virtual ids -- must not be
|
||||
recorded, or each replay overwrites the translated tables."""
|
||||
"""A translating fa3 must not record the in-graph draft-extend gather: it
|
||||
reads raw (virtual) req_to_token ids over the tables rebuilt out of graph."""
|
||||
|
||||
def _run(self, *, translating):
|
||||
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
|
||||
@@ -140,10 +135,8 @@ class TestDraftExtendInGraph(CustomTestCase):
|
||||
fb = SimpleNamespace(forward_mode=ForwardMode.DRAFT_EXTEND_V2, batch_size=2)
|
||||
b.init_forward_metadata_in_graph(fb)
|
||||
|
||||
def test_translating_backend_records_nothing(self):
|
||||
def test_only_a_static_backend_builds_in_graph(self):
|
||||
self._run(translating=True)
|
||||
|
||||
def test_static_backend_still_builds_in_graph(self):
|
||||
with self.assertRaises(KeyError):
|
||||
self._run(translating=False)
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Nothing under layers/attention may translate KV ids for itself.
|
||||
|
||||
Ownership is exactly two places: `KVIndexTranslator` for READS and the
|
||||
ForwardBatch rebind (`rebind_write_loc`) for WRITES. Virtual and physical ids
|
||||
share a value range, so a backend that forgets a translate -- or does one
|
||||
twice -- reads the wrong rows and nothing crashes.
|
||||
Ownership is exactly one place: the iteration's `KVLocPlan`, built through
|
||||
`KVIndexTranslator`, translates the WRITE window and the READ table once, and
|
||||
backends read the result. Virtual and physical ids share a value range, so a
|
||||
backend that forgets a translate -- or does one twice -- reads the wrong rows
|
||||
and nothing crashes.
|
||||
|
||||
Deliberately out of scope: the allocator-internal implementations
|
||||
(`allocator/unified_*`, `unified_memory_pool`), which ARE the mechanism the
|
||||
|
||||
@@ -9,6 +9,7 @@ from sglang.srt.layers.attention.triton_backend import (
|
||||
TritonAttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -26,16 +27,19 @@ class _BonusTokenVerifyInput(SpecInput):
|
||||
class _KVIndexTranslator:
|
||||
is_translating = False
|
||||
|
||||
def fill_packed_read_stream(
|
||||
def pack_read_stream(
|
||||
self,
|
||||
plan,
|
||||
*,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
indptr,
|
||||
total_tokens,
|
||||
out,
|
||||
kv_start_idx=None,
|
||||
kind=IdSpaceKind.FULL,
|
||||
):
|
||||
out.zero_()
|
||||
return False
|
||||
|
||||
|
||||
class _RecordingTritonBackend(TritonAttnBackend):
|
||||
@@ -92,6 +96,8 @@ def _make_forward_batch(batch_size, spec_info):
|
||||
spec_info=spec_info,
|
||||
forward_mode=ForwardMode.TARGET_VERIFY,
|
||||
out_cache_loc=torch.zeros(batch_size * 7, dtype=torch.int64),
|
||||
# A static pool's reads never touch the plan.
|
||||
kv_loc_plan=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -15,14 +15,10 @@
|
||||
|
||||
A whole-sequence verify reads `[committed prefix | drafts]` back out of the
|
||||
pool, so under the unified pool the translated page table a verify kernel
|
||||
reads has to cover `seq_lens + num_draft_tokens` columns per row. A table
|
||||
filled only to `seq_lens` leaves the draft tail stale: invisible at
|
||||
page_size 256, where the un-widened `ceil(seq/ps)` pages still cover a few
|
||||
draft tokens, and at page_size 1 observed as accept length collapsing
|
||||
(6.20 -> 1.37) or wrong tokens (gsm8k 0.905 -> 0.730).
|
||||
reads has to cover `seq_lens + num_draft_tokens` columns per row; a table
|
||||
filled only to `seq_lens` silently leaves the draft tail stale.
|
||||
|
||||
These tests drive fa3's two verify builds -- the eager one and the captured
|
||||
one a cuda-graph replay runs -- through a translating KV-index translator
|
||||
These tests drive fa3's eager and captured verify builds from a verify plan
|
||||
over a non-identity virtual->physical page map, and check the page table the
|
||||
kernel reads: every row translated, and filled through the draft tail.
|
||||
|
||||
@@ -39,6 +35,7 @@ from sglang.srt.layers.attention.flashattention_backend import (
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -67,9 +64,16 @@ def _translator(req_to_token, page_size, n_pages):
|
||||
t.defer_read_translate = False
|
||||
t._capture_page_size = page_size
|
||||
t._full_v2p_table = v2p.to(_DEV)
|
||||
t._swa_v2p_table = None
|
||||
t._spaces = {
|
||||
IdSpaceKind.FULL: IdSpace(
|
||||
key=(IdSpaceKind.FULL, "test"),
|
||||
write=lambda ids: (
|
||||
t._full_v2p_table[ids // page_size] * page_size + ids % page_size
|
||||
),
|
||||
read_v2p=t._full_v2p_table,
|
||||
)
|
||||
}
|
||||
t._rows = torch.arange(req_to_token.shape[0], dtype=torch.int64, device=_DEV)
|
||||
t._index_table_memo = None
|
||||
return t
|
||||
|
||||
|
||||
@@ -99,6 +103,25 @@ def _expected(req_to_token, live, v2p, page_size):
|
||||
return rows
|
||||
|
||||
|
||||
def _verify_plan(translator, req_to_token, live, host_lens=None):
|
||||
"""The plan a verify forward builds for itself: it writes the draft window
|
||||
past `live`, so it reads that far."""
|
||||
bs = live.numel()
|
||||
rows = torch.arange(bs, device=_DEV)
|
||||
cols = live.to(_DEV)[:, None] + torch.arange(_NUM_DRAFT, device=_DEV)[None, :]
|
||||
return translator.own_plan(
|
||||
SimpleNamespace(
|
||||
forward_mode=ForwardMode.TARGET_VERIFY,
|
||||
spec_info=SimpleNamespace(draft_token_num=_NUM_DRAFT),
|
||||
batch_size=bs,
|
||||
req_pool_indices=rows,
|
||||
seq_lens=live.to(_DEV),
|
||||
seq_lens_cpu=live if host_lens is None else host_lens,
|
||||
out_cache_loc=req_to_token[rows[:, None], cols].long().reshape(-1),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _backend(translator, req_to_token, page_size):
|
||||
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
|
||||
b.kv_index_translator = translator
|
||||
@@ -153,6 +176,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase):
|
||||
metadata.swa_page_table = None
|
||||
b.target_verify_metadata = {bs: metadata}
|
||||
b._apply_cuda_graph_metadata(
|
||||
_verify_plan(translator, req_to_token, live),
|
||||
bs,
|
||||
torch.arange(bs, device=_DEV),
|
||||
live.to(_DEV),
|
||||
@@ -193,6 +217,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase):
|
||||
spec_info=spec_info,
|
||||
encoder_lens=None,
|
||||
out_cache_loc=None,
|
||||
kv_loc_plan=_verify_plan(translator, req_to_token, live, host_lens),
|
||||
)
|
||||
b.init_forward_metadata(fb)
|
||||
self._check(
|
||||
|
||||
@@ -225,7 +225,12 @@ class TestMambaPrefillTrackMetadata(unittest.TestCase):
|
||||
model_config=SimpleNamespace(
|
||||
requires_mm_token_modalities=False, model_is_mrope=False
|
||||
),
|
||||
kv_index_translator=SimpleNamespace(rebind_write_loc=lambda forward: None),
|
||||
# A pass-through runner: its plan hands the batch its ids untouched.
|
||||
kv_index_translator=SimpleNamespace(
|
||||
own_plan=lambda batch: SimpleNamespace(
|
||||
bind=lambda batch, reader, cols=None: None
|
||||
)
|
||||
),
|
||||
prefill_attention_backend_str="torch_native",
|
||||
ngram_embedding_manager=SimpleNamespace(enabled=False),
|
||||
lora_manager=None,
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
"""Dense fused-draft geometry: the draft model's KV as two more parts of every
|
||||
host slot's entry.
|
||||
|
||||
Derived-property pins for the fused entry layout
|
||||
host slot's entry,
|
||||
|
||||
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
|
||||
|
||||
the draft parts start at the 16-B-aligned end of the host parts and the entry
|
||||
rounds up to the 32-B entry alignment (never to an lcm of row widths), so the
|
||||
draft's row width is free to differ from the host's; host and draft views
|
||||
share the slot stride and are indexed by the same physical token id; and
|
||||
the draft parts starting at the aligned end of the host parts. Host and draft
|
||||
views share the slot stride and are indexed by the same physical token id;
|
||||
writes through either family's views land inside their own part of their own
|
||||
slot (compaction moves whole page envelopes, so confinement IS the correctness
|
||||
of the fused move).
|
||||
slot, and a host page move carries the draft bytes with it.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py -v
|
||||
"""
|
||||
@@ -27,7 +22,6 @@ from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
)
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
ENTRY_ALIGN_BYTES,
|
||||
ROW_ALIGN_BYTES,
|
||||
align_entry_bytes,
|
||||
align_part_offset,
|
||||
)
|
||||
@@ -43,11 +37,10 @@ from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=7, suite="base-a-test-cpu")
|
||||
|
||||
_DEV = "cpu"
|
||||
_DTYPE = torch.bfloat16
|
||||
_ITEM = _DTYPE.itemsize
|
||||
|
||||
|
||||
def _host_spec(draft_region=None, layer_num=2, head_num=2, head_dim=4):
|
||||
@@ -75,12 +68,6 @@ def _placement(region):
|
||||
|
||||
|
||||
class TestFusedSpecMath(unittest.TestCase):
|
||||
def test_unfused_spec_is_byte_identical_to_before(self):
|
||||
s = _host_spec()
|
||||
self.assertEqual(s.entry_bytes(), 2 * (16 + 16))
|
||||
self.assertEqual(s.host_entry_bytes(), s.entry_bytes())
|
||||
self.assertEqual([p.name for p in s.layout().parts], ["k", "v"])
|
||||
|
||||
def test_fused_entry_appends_the_draft_parts(self):
|
||||
f = _host_spec(_draft_region())
|
||||
r = f.draft_region
|
||||
@@ -101,46 +88,6 @@ class TestFusedSpecMath(unittest.TestCase):
|
||||
self.assertEqual(layout.part("draft_v").offset_bytes, 64 + 48)
|
||||
self.assertEqual(layout.part("draft_k").layer_stride_bytes, 64)
|
||||
|
||||
def test_entry_alignment_pads_only_to_the_entry_quantum(self):
|
||||
# host 32 B + draft 48 B = 80 B of rows -> one 96 B entry (16 B pad),
|
||||
# not an lcm of the two row widths.
|
||||
f = _host_spec(
|
||||
DenseDraftRegion(
|
||||
lane_num=1, head_num=1, head_dim=8, v_head_dim=16, store_dtype=_DTYPE
|
||||
),
|
||||
layer_num=1,
|
||||
head_num=1,
|
||||
head_dim=8,
|
||||
)
|
||||
self.assertEqual(f.host_entry_bytes(), 32)
|
||||
self.assertEqual(f.draft_region.entry_bytes(), 48)
|
||||
self.assertEqual(f.entry_bytes(), 96)
|
||||
self.assertEqual(f.draft_offset_in_entry() % ROW_ALIGN_BYTES, 0)
|
||||
f.layout() # parts fit and do not overlap
|
||||
|
||||
def test_draft_rows_must_be_row_aligned(self):
|
||||
with self.assertRaises(AssertionError):
|
||||
_host_spec(
|
||||
DenseDraftRegion(lane_num=1, head_num=1, head_dim=3, store_dtype=_DTYPE)
|
||||
).layout()
|
||||
|
||||
def test_only_page_envelope_kinds_accept_a_draft_region(self):
|
||||
"""`draft_region` is a universal spec field (None = unfused), but a
|
||||
kind without the fused entry layout must refuse one at construction;
|
||||
a silently-carried region would never reach the layout. MHA and MLA
|
||||
entries carry the draft parts; mamba state pages carry none yet."""
|
||||
with self.assertRaises(AssertionError):
|
||||
MambaSubPoolSpec(
|
||||
name="mamba",
|
||||
layer_num=1,
|
||||
conv_state_shapes=((2, 2),),
|
||||
conv_dtype=torch.float32,
|
||||
temporal_state_shape=(2,),
|
||||
temporal_dtype=torch.float32,
|
||||
grow_direction="up",
|
||||
draft_region=_draft_region(),
|
||||
)
|
||||
|
||||
|
||||
class TestFusedRegionConfinement(unittest.TestCase):
|
||||
"""Writes through host/draft views must land in their own part of their
|
||||
@@ -199,20 +146,6 @@ class TestFusedRegionConfinement(unittest.TestCase):
|
||||
self.assertGreaterEqual(lo, page * self.PS * entry)
|
||||
self.assertLessEqual(hi, (page + 1) * self.PS * entry)
|
||||
|
||||
def test_host_and_draft_views_share_the_slot_stride(self):
|
||||
spec, _, (hk, hv), (dk, dv) = self._build()
|
||||
stride = spec.entry_bytes()
|
||||
for view in (*hk, *hv, *dk, *dv):
|
||||
self.assertEqual(view.stride(0) * view.element_size(), stride)
|
||||
self.assertEqual(view.shape[0], hk[0].shape[0])
|
||||
self.assertEqual(tuple(dk[0].shape[1:]), (1, 24))
|
||||
self.assertEqual(tuple(dv[0].shape[1:]), (1, 8))
|
||||
|
||||
def test_pool_helper_refuses_an_unfused_sub_pool(self):
|
||||
_, pool, _, _ = self._build()
|
||||
with self.assertRaises(AssertionError):
|
||||
pool.build_dense_draft_views("swa") # no fused region there
|
||||
|
||||
|
||||
class TestUnifiedDraftKVPool(unittest.TestCase):
|
||||
PS = 2
|
||||
@@ -268,9 +201,9 @@ class TestUnifiedDraftKVPool(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_host_page_move_carries_the_draft_bytes(self):
|
||||
# THE fused-layout property: compaction relocates whole page envelopes
|
||||
# on the HOST pool; a draft marker written in page A must arrive at
|
||||
# page B after host.move_kv_cache(B, A), with zero draft-side moves.
|
||||
# Compaction relocates whole page envelopes on the HOST pool: a draft
|
||||
# marker written in page A arrives at page B after
|
||||
# host.move_kv_cache(B, A), with no draft-side move.
|
||||
pool = self._pool()
|
||||
dp, _ = self._draft_pool(pool)
|
||||
host = UnifiedMHATokenToKVPool(
|
||||
@@ -336,17 +269,6 @@ class TestUnifiedDraftKVPool(unittest.TestCase):
|
||||
self.assertEqual(dp.get_key_buffer(0).dtype, fp8)
|
||||
torch.testing.assert_close(dp.get_key_buffer(0)[loc].float(), k)
|
||||
|
||||
def test_draft_side_moves_and_transfers_fail_loudly(self):
|
||||
pool = self._pool()
|
||||
dp, _ = self._draft_pool(pool)
|
||||
one = torch.zeros(self.PS, dtype=torch.int64)
|
||||
with self.assertRaises(NotImplementedError):
|
||||
dp.move_kv_cache(one, one)
|
||||
with self.assertRaises(NotImplementedError):
|
||||
dp.get_contiguous_buf_infos()
|
||||
with self.assertRaises(NotImplementedError):
|
||||
dp.get_cpu_copy(one)
|
||||
|
||||
|
||||
def _mla_host_spec(draft_region=None):
|
||||
# 32 B latent rows, two layers: a 64 B host entry before the draft parts.
|
||||
@@ -376,8 +298,7 @@ def _mamba_spec():
|
||||
class TestFusedMLAHost(unittest.TestCase):
|
||||
"""MLA entries carry the fused draft parts exactly like MHA entries: the
|
||||
same aligned entry, one slot stride for both families, and byte-disjoint
|
||||
host/draft parts within every slot. The unfused spec stays byte-identical;
|
||||
that identity guards every existing MLA deploy."""
|
||||
host/draft parts within every slot."""
|
||||
|
||||
def setUp(self):
|
||||
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
|
||||
@@ -389,13 +310,6 @@ class TestFusedMLAHost(unittest.TestCase):
|
||||
PS = 2
|
||||
PAGES = 8
|
||||
|
||||
def test_unfused_spec_is_byte_identical_to_before(self):
|
||||
s = _mla_host_spec()
|
||||
self.assertEqual(s.row_bytes(), 32)
|
||||
self.assertEqual(s.entry_bytes(), 64)
|
||||
self.assertEqual(s.host_entry_bytes(), s.entry_bytes())
|
||||
self.assertEqual([p.name for p in s.layout().parts], ["kv"])
|
||||
|
||||
def test_fused_entry_appends_the_draft_parts(self):
|
||||
f = _mla_host_spec(_draft_region())
|
||||
self.assertEqual(f.host_entry_bytes(), 64)
|
||||
@@ -448,10 +362,9 @@ class TestFusedMLAHost(unittest.TestCase):
|
||||
self.assertTrue(bool((nz >= split).all() and (nz < slot_hi).all()))
|
||||
|
||||
def test_translator_takes_the_fused_disposition_on_the_mla_host(self):
|
||||
"""A draft runner bound over the MLA host's entries must translate
|
||||
through the mamba allocator's full-side v2p, exactly as on the SWA
|
||||
host. A kind-specific probe regression here silently reverts the
|
||||
draft to passthrough (raw virtual ids into the views)."""
|
||||
"""A draft runner bound over the MLA host's entries translates through
|
||||
the mamba allocator's full-side v2p, as on the SWA host; a passthrough
|
||||
here would write raw virtual ids into the views."""
|
||||
from sglang.srt.mem_cache.allocator.unified_mamba import (
|
||||
UnifiedMambaTokenToKVPoolAllocator,
|
||||
)
|
||||
@@ -489,7 +402,7 @@ class TestFusedMLAHost(unittest.TestCase):
|
||||
)
|
||||
self.assertTrue(translator.is_translating)
|
||||
self.assertIs(
|
||||
translator.full_flat_v2p(), alloc.full_attn_allocator.virtual_to_physical
|
||||
translator.full_v2p_table, alloc.full_attn_allocator.virtual_to_physical
|
||||
)
|
||||
|
||||
def test_draft_pool_binds_over_the_mla_host(self):
|
||||
|
||||
@@ -280,9 +280,9 @@ class TestHybridLinearMLARouting(unittest.TestCase):
|
||||
return pool
|
||||
|
||||
def test_set_mla_kv_buffer_door_never_translates(self):
|
||||
"""The translate happens exactly once, at ForwardBatch construction
|
||||
(`rebind_write_loc`); a door that translated again would
|
||||
double-translate every unified MLA write."""
|
||||
"""The translate happens exactly once, in the iteration's plan
|
||||
(`KVLocPlan`); a door that translated again would double-translate
|
||||
every unified MLA write."""
|
||||
pool = self._make_bare_pool()
|
||||
loc = torch.tensor([107, 108, 109], dtype=torch.int64)
|
||||
layer = types.SimpleNamespace(layer_id=0)
|
||||
|
||||
@@ -11,29 +11,24 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`place_fused_draft`: which host/draft pairs fuse, and where each runner's
|
||||
layers land.
|
||||
"""`place_fused_draft` and the target's fused-draft decision: which drafts
|
||||
fuse into the host's entries, and where each runner's layers land.
|
||||
|
||||
A decline keeps a DFLASH or DSPARK draft on its private pool (a DFLASH draft
|
||||
on a mamba host is refused instead), and refuses an EAGLE draft at boot. A
|
||||
wrong answer for a draft that keeps its pool silently changes what it binds
|
||||
and what the boot solve prices. Pinned:
|
||||
A decline keeps a draft on a private pool (or refuses it at boot); admitting
|
||||
one that cannot read the fused rows correctly gives silently wrong drafts.
|
||||
Pinned:
|
||||
- a replicated head under multi-layer EAGLE gets one lane RANGE per runner
|
||||
(one shared region would let the runners clobber each other's KV);
|
||||
- a per-depth head serves one depth per runner and needs one runner per
|
||||
depth;
|
||||
- a draft with SWA or recurrent-state layers of its own, or asymmetric K/V
|
||||
rows, declines: the fused arm binds one dense pool over the host's full
|
||||
slots;
|
||||
- a draft whose attention backend is off the translated MHA rails
|
||||
declines: it would read the fused rows without the KV-index translator;
|
||||
- so does a draft under --dcp-size > 1, where each rank's host rows hold
|
||||
only its share of the tokens the replicated draft reads, and a draft
|
||||
under HiCache, which builds its host pool off a device pool of its own;
|
||||
- an explicit draft KV dtype unlike the host's declines;
|
||||
- the profile divides the draft's heads by attn_tp, as the target does;
|
||||
- a placement whose runner lane counts do not fill its region is refused;
|
||||
- the priced entry counts the layers THIS runner owns, not the whole model's.
|
||||
rows, declines;
|
||||
- so does a draft whose attention backend is off the translated MHA rails
|
||||
(it would read the fused rows with virtual ids), a draft under
|
||||
--dcp-size > 1 (each rank's rows hold only its share of the tokens), an
|
||||
explicit draft KV dtype unlike the host's, and a draft under HiCache or
|
||||
host-pool decode retraction (they build the draft's host pool off a
|
||||
device pool of its own).
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v
|
||||
"""
|
||||
@@ -48,9 +43,6 @@ from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
DenseDraftRegion,
|
||||
DraftKVGeometry,
|
||||
DraftKVProfile,
|
||||
FusedDraftDecision,
|
||||
FusedDraftPlacement,
|
||||
draft_kv_profile,
|
||||
place_fused_draft,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, override_platform
|
||||
@@ -130,118 +122,6 @@ class TestPlaceFusedDraft(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestDraftKVProfile(CustomTestCase):
|
||||
def test_heads_are_divided_by_attn_tp(self):
|
||||
mc = SimpleNamespace(
|
||||
is_hybrid_swa=True,
|
||||
is_deepseek_v4_arch=False,
|
||||
swa_attention_layer_ids=[0],
|
||||
num_nextn_predict_layers=None,
|
||||
get_num_kv_heads=lambda tp: max(1, 8 // tp),
|
||||
head_dim=64,
|
||||
v_head_dim=32,
|
||||
)
|
||||
with patch("sglang.srt.configs.hybrid_arch.mambaish_config", return_value=None):
|
||||
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=2)
|
||||
self.assertEqual(
|
||||
profile.full, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32)
|
||||
)
|
||||
self.assertEqual(profile.swa_layer_ids, (0,))
|
||||
self.assertEqual(profile.num_depths, 1)
|
||||
self.assertEqual(profile.num_state_layers, 0)
|
||||
|
||||
def _linear_trunk_mc(self, hf_text_config):
|
||||
return SimpleNamespace(
|
||||
is_hybrid_swa=False,
|
||||
is_deepseek_v4_arch=False,
|
||||
num_nextn_predict_layers=1,
|
||||
get_num_kv_heads=lambda tp: 8,
|
||||
head_dim=64,
|
||||
v_head_dim=64,
|
||||
hf_text_config=hf_text_config,
|
||||
)
|
||||
|
||||
def test_a_nextn_head_of_a_linear_trunk_owns_no_state(self):
|
||||
"""BUG REGRESSION. A NEXTN head ships inside the trunk checkpoint and
|
||||
inherits its config CLASS, so `mamba2_cache_params` lists the TRUNK's
|
||||
state layers while the head is a full-attention block. Counting them
|
||||
declined a fusable draft to its private pool."""
|
||||
mc = self._linear_trunk_mc(SimpleNamespace())
|
||||
trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2]))
|
||||
with patch(
|
||||
"sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk
|
||||
):
|
||||
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1)
|
||||
self.assertEqual(profile.num_state_layers, 0)
|
||||
|
||||
def test_a_conv_chain_head_still_counts_its_state(self):
|
||||
"""The discriminator must not silence a head that really owns state:
|
||||
only a conv-chain MTP config declares `mtp_local_layer_ids`."""
|
||||
mc = self._linear_trunk_mc(SimpleNamespace(mtp_local_layer_ids=[0]))
|
||||
trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2]))
|
||||
with patch(
|
||||
"sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk
|
||||
):
|
||||
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1)
|
||||
self.assertEqual(profile.num_state_layers, 3)
|
||||
|
||||
|
||||
class TestFusedDraftPlacement(CustomTestCase):
|
||||
def test_runner_lane_counts_must_fill_the_region(self):
|
||||
region = DenseDraftRegion(
|
||||
lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE
|
||||
)
|
||||
with self.assertRaises(AssertionError):
|
||||
FusedDraftPlacement(region=region, runner_lane_counts=(1,))
|
||||
with self.assertRaises(AssertionError):
|
||||
FusedDraftPlacement(region=region, runner_lane_counts=())
|
||||
|
||||
|
||||
class TestFusedEntryPricing(CustomTestCase):
|
||||
"""The boot solve prices a fused entry through the same spec the pool
|
||||
factory builds, so the two cannot drift. The factory builds the full
|
||||
sub-pool from this runner's OWN layer slice; pricing from the whole-model
|
||||
split over-counts every layer another pipeline rank holds, and the solve
|
||||
then hands out fewer tokens than fit."""
|
||||
|
||||
def _configurator(self, *, whole, hybrid_swa, owned=(), span=(0, 0)):
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
|
||||
cfg = KVCacheConfigurator.__new__(KVCacheConfigurator)
|
||||
cfg.is_hybrid_swa = hybrid_swa
|
||||
cfg.layer_info = SimpleNamespace(
|
||||
full_attention_layer_ids=list(owned),
|
||||
start_layer=span[0],
|
||||
end_layer=span[1],
|
||||
)
|
||||
cfg.mambaish_config = SimpleNamespace(full_attention_layer_ids=whole)
|
||||
cfg.model_config = SimpleNamespace(
|
||||
full_attention_layer_ids=whole,
|
||||
head_dim=8,
|
||||
get_num_kv_heads=lambda *_: 1,
|
||||
)
|
||||
cfg.use_mla_backend = False
|
||||
cfg.kv_cache_dtype = _DTYPE
|
||||
return cfg
|
||||
|
||||
def _priced(self, cfg):
|
||||
region = DenseDraftRegion(
|
||||
lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE
|
||||
)
|
||||
with get_parallel().override(attn_tp_size=1, attn_dcp_size=1):
|
||||
return cfg._full_host_spec(region).layer_num
|
||||
|
||||
def test_an_swa_host_prices_this_runners_slice(self):
|
||||
cfg = self._configurator(
|
||||
whole=[0, 1, 2, 3], hybrid_swa=True, owned=[0, 1], span=(0, 2)
|
||||
)
|
||||
self.assertEqual(self._priced(cfg), 2)
|
||||
|
||||
def test_a_mamba_host_prices_only_the_layers_in_its_span(self):
|
||||
cfg = self._configurator(whole=[0, 1, 2, 3], hybrid_swa=False, span=(2, 4))
|
||||
self.assertEqual(self._priced(cfg), 2)
|
||||
|
||||
|
||||
class TestFusedDraftDecision(CustomTestCase):
|
||||
"""The target's boot decision over a host whose full sub-pool fuses."""
|
||||
|
||||
@@ -252,7 +132,6 @@ class TestFusedDraftDecision(CustomTestCase):
|
||||
draft_kv_dtype=None,
|
||||
host_kv_dtype=_DTYPE,
|
||||
kv_cache_dtype_flag="auto",
|
||||
attention_arch=None,
|
||||
draft_backend=None,
|
||||
target_backends=("triton", "triton"),
|
||||
dcp_size=1,
|
||||
@@ -286,7 +165,7 @@ class TestFusedDraftDecision(CustomTestCase):
|
||||
head_dim=64,
|
||||
v_head_dim=64,
|
||||
dtype=torch.bfloat16,
|
||||
attention_arch=attention_arch or AttentionArch.MHA,
|
||||
attention_arch=AttentionArch.MHA,
|
||||
),
|
||||
)
|
||||
memory = SimpleNamespace(
|
||||
@@ -326,15 +205,6 @@ class TestFusedDraftDecision(CustomTestCase):
|
||||
def test_a_stateless_draft_is_placed(self):
|
||||
self.assertIsNotNone(self._decide().placement)
|
||||
|
||||
def test_a_non_mha_draft_keeps_the_private_pool(self):
|
||||
"""The region holds dense MHA K/V rows. An MLA draft is kept out by its
|
||||
own architecture, not by the accident of asymmetric head dims."""
|
||||
from sglang.srt.configs.model_config import AttentionArch
|
||||
|
||||
declined = self._decide(attention_arch=AttentionArch.MLA)
|
||||
self.assertIsNone(declined.placement)
|
||||
self.assertIn("MLA", declined.declined)
|
||||
|
||||
def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self):
|
||||
"""A fused draft stores its rows in the host's KV dtype; an explicit
|
||||
draft dtype that differs declines instead of mis-typing those rows,
|
||||
@@ -400,90 +270,5 @@ class TestFusedDraftDecision(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestMambaHostPrivateDraftRefused(CustomTestCase):
|
||||
"""A mamba host's unified buffer takes the whole KV budget, so a private
|
||||
draft pool on top of it would overcommit; the fused arm is the only
|
||||
EAGLE-family or DFLASH arm such a host builds. A DSPARK draft that does
|
||||
not fuse keeps its private pool."""
|
||||
|
||||
_DECLINED = "the draft's K/V rows are asymmetric"
|
||||
|
||||
def _resolve(self, *, decision, algorithm="EAGLE"):
|
||||
from sglang.srt.mem_cache import kv_cache_configurator as kvc
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator)
|
||||
cfg.is_draft_worker = False
|
||||
cfg.spec_algorithm = SpeculativeAlgorithm[algorithm]
|
||||
with patch.object(
|
||||
kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision
|
||||
):
|
||||
return cfg._fused_draft_for_mamba_factory()
|
||||
|
||||
def test_a_declined_eagle_draft_is_refused(self):
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
with self.subTest(algorithm=algorithm):
|
||||
with self.assertRaisesRegex(ValueError, "rows are asymmetric"):
|
||||
self._resolve(
|
||||
decision=FusedDraftDecision(declined=self._DECLINED),
|
||||
algorithm=algorithm,
|
||||
)
|
||||
|
||||
def test_a_declined_dflash_draft_is_refused_and_dspark_keeps_its_pool(self):
|
||||
"""A DFLASH decliner is refused like EAGLE's; a DSPARK decliner keeps
|
||||
its private pool instead of failing the boot."""
|
||||
declined = FusedDraftDecision(declined=self._DECLINED)
|
||||
with self.assertRaisesRegex(ValueError, "rows are asymmetric"):
|
||||
self._resolve(decision=declined, algorithm="DFLASH")
|
||||
self.assertIsNone(self._resolve(decision=declined, algorithm="DSPARK"))
|
||||
|
||||
def test_a_placed_draft_and_other_algorithms_pass(self):
|
||||
placement = _place(_profile()).placement
|
||||
for algorithm in ("EAGLE", "DFLASH", "DSPARK"):
|
||||
with self.subTest(algorithm=algorithm):
|
||||
self.assertIs(
|
||||
self._resolve(
|
||||
decision=FusedDraftDecision(placement=placement),
|
||||
algorithm=algorithm,
|
||||
),
|
||||
placement,
|
||||
)
|
||||
self.assertIsNone(
|
||||
self._resolve(decision=FusedDraftDecision(), algorithm="NONE")
|
||||
)
|
||||
|
||||
|
||||
class TestSWAHostPrivateDraftRefused(CustomTestCase):
|
||||
"""A hybrid-SWA host's boot solve prices a private EAGLE draft at the
|
||||
target's per-token size, so a draft that does not fuse is refused there
|
||||
too; the fused arm is the only EAGLE arm such a host builds."""
|
||||
|
||||
def _resolve(self, *, decision, eagle=True):
|
||||
from sglang.srt.mem_cache import kv_cache_configurator as kvc
|
||||
|
||||
cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator)
|
||||
cfg.is_draft_worker = False
|
||||
cfg.spec_algorithm = SimpleNamespace(is_eagle=lambda: eagle)
|
||||
with patch.object(
|
||||
kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision
|
||||
):
|
||||
return cfg._fused_draft_for_swa_factory()
|
||||
|
||||
def test_a_declined_eagle_draft_is_refused(self):
|
||||
with self.assertRaisesRegex(ValueError, "KV cache dtype"):
|
||||
self._resolve(
|
||||
decision=FusedDraftDecision(
|
||||
declined="the draft's KV cache dtype (bf16) differs from the host's"
|
||||
)
|
||||
)
|
||||
|
||||
def test_a_placed_draft_and_other_algorithms_pass(self):
|
||||
placement = _place(_profile()).placement
|
||||
self.assertIs(
|
||||
self._resolve(decision=FusedDraftDecision(placement=placement)), placement
|
||||
)
|
||||
self.assertIsNone(self._resolve(decision=FusedDraftDecision(), eagle=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -11,20 +11,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Draft-shaped KVCacheConfigurator dispatch under the unified pool.
|
||||
"""How a draft worker's KVCacheConfigurator binds its KV under the unified pool.
|
||||
|
||||
BUG REGRESSION (fast path). The fast path was gated on `req_to_token_pool is
|
||||
None` alone, but a compact-window DFLASH draft (`--speculative-draft-window-size`)
|
||||
also passes None — it builds a private req_to_token of its own — so the draft
|
||||
worker would allocate a SECOND unified byte buffer at boot. A draft-shaped
|
||||
configurator must fall through to the normal pool build instead.
|
||||
|
||||
DERIVED PROPERTY (binding dispatch). The fused-vs-private draft binding
|
||||
dispatches on the spec algorithm, not the allocator kind: a non-EAGLE draft on
|
||||
a unified SWA target takes the private arm, sized by the full sub-allocator's
|
||||
VIRTUAL id space (`max_slots - 1`). The SWA allocator's `size_full` reports
|
||||
the static token budget — smaller than the id space — so sizing by it would
|
||||
put verify-window writes at high virtual ids out of bounds.
|
||||
With a fused region the draft binds `UnifiedDraftKVPool` over the target's
|
||||
allocator (a compact-window DFLASH draft with a private `req_to_token` of its
|
||||
own), refusing a KV dtype unlike the region's. Without one it takes the private
|
||||
arm, sized by the full sub-allocator's VIRTUAL id space (`max_slots - 1`), not
|
||||
`size_full`, the smaller static token budget: sizing by that would put
|
||||
verify-window writes at high virtual ids out of bounds.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v
|
||||
"""
|
||||
@@ -56,82 +50,7 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _ReachedNormalBuild(Exception):
|
||||
"""Sentinel: control flow fell past the unified fast path."""
|
||||
|
||||
|
||||
class TestUnifiedFastPathDraftGate(CustomTestCase):
|
||||
def _run(self, *, is_draft_worker: bool):
|
||||
cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator)
|
||||
cfg.is_draft_worker = is_draft_worker
|
||||
cfg.mambaish_config = object() # would select the mamba unified arm
|
||||
cfg.is_hybrid_swa = False # not the mamba+SWA tri-pool arm
|
||||
cfg.model_config = SimpleNamespace(hf_config={}) # not DSV4
|
||||
# Every field the fast path reads: a missing one raises while the
|
||||
# arm's arguments are evaluated, which reads here as "took the fast
|
||||
# path but called nothing".
|
||||
sizes = SimpleNamespace(
|
||||
max_running_requests=8,
|
||||
max_total_num_tokens=64,
|
||||
full_max_total_num_tokens=64,
|
||||
swa_max_total_num_tokens=32,
|
||||
unified_total_bytes=1 << 20,
|
||||
)
|
||||
taken = []
|
||||
with (
|
||||
patch.object(
|
||||
kcc,
|
||||
"get_memory",
|
||||
return_value=SimpleNamespace(enable_unified_memory=True),
|
||||
),
|
||||
patch.object(
|
||||
kcc,
|
||||
"get_disagg",
|
||||
return_value=SimpleNamespace(disaggregation_mode="null"),
|
||||
),
|
||||
patch.object(
|
||||
kcc.KVCacheConfigurator,
|
||||
"_init_unified_mamba_pools",
|
||||
lambda self, **kw: taken.append("mamba") or None,
|
||||
),
|
||||
patch.object(
|
||||
kcc.KVCacheConfigurator,
|
||||
"_init_unified_swa_pools",
|
||||
lambda self, **kw: taken.append("swa") or None,
|
||||
),
|
||||
patch.object(
|
||||
kcc.KVCacheConfigurator,
|
||||
"_build_req_to_token_pool",
|
||||
side_effect=_ReachedNormalBuild,
|
||||
),
|
||||
):
|
||||
try:
|
||||
cfg._init_pools(
|
||||
sizes=sizes,
|
||||
req_to_token_pool=None,
|
||||
token_to_kv_pool_allocator=None,
|
||||
)
|
||||
except _ReachedNormalBuild:
|
||||
return "normal", taken
|
||||
except (AttributeError, TypeError):
|
||||
# The stubbed unified arm returned None; anything past the
|
||||
# fast-path dispatch counts as having taken it.
|
||||
return "unified", taken
|
||||
return "unified", taken
|
||||
|
||||
def test_draft_worker_falls_through_to_the_normal_build(self):
|
||||
path, taken = self._run(is_draft_worker=True)
|
||||
self.assertEqual(path, "normal")
|
||||
self.assertEqual(taken, [])
|
||||
|
||||
def test_target_worker_still_takes_the_fast_path(self):
|
||||
"""The guard must narrow to drafts only — a target regression here
|
||||
silently turns --enable-unified-memory into a no-op."""
|
||||
path, taken = self._run(is_draft_worker=False)
|
||||
self.assertEqual(taken, ["mamba"])
|
||||
register_cpu_ci(est_time=5, stage="weekly", runner_config="cpu")
|
||||
|
||||
|
||||
class _FakeKVCache:
|
||||
@@ -240,7 +159,6 @@ class TestDraftBindingDispatch(CustomTestCase):
|
||||
alloc,
|
||||
max_total_num_tokens,
|
||||
kv_dtype=torch.bfloat16,
|
||||
state_layers=(),
|
||||
req_to_token_pool="shared",
|
||||
):
|
||||
if req_to_token_pool == "shared":
|
||||
@@ -295,9 +213,7 @@ class TestDraftBindingDispatch(CustomTestCase):
|
||||
# The real scan walks the draft nn.Module for RadixAttention layers.
|
||||
patch.object(unified_draft_pool, "draft_kv_layer_ids", return_value=[0]),
|
||||
patch.object(
|
||||
unified_draft_pool,
|
||||
"draft_state_layer_classes",
|
||||
return_value=list(state_layers),
|
||||
unified_draft_pool, "draft_state_layer_classes", return_value=[]
|
||||
),
|
||||
patch.object(
|
||||
kcc.KVCacheConfigurator,
|
||||
@@ -325,34 +241,33 @@ class TestDraftBindingDispatch(CustomTestCase):
|
||||
sized = caught.exception.sizes.max_total_num_tokens
|
||||
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
|
||||
|
||||
def test_eagle_draft_still_binds_the_fused_pool(self):
|
||||
def test_a_draft_with_a_region_binds_the_fused_pool(self):
|
||||
"""An EAGLE or DSPARK draft binds the fused pool over the target's
|
||||
allocator when the target resolved a region; the region-less private
|
||||
arm above is the fallback."""
|
||||
alloc = self._swa_allocator(with_draft_region=True)
|
||||
pools = self._run(
|
||||
algorithm=SpeculativeAlgorithm.EAGLE3,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
)
|
||||
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
|
||||
self.assertIs(pools.token_to_kv_pool_allocator, alloc)
|
||||
for algorithm in (SpeculativeAlgorithm.EAGLE3, SpeculativeAlgorithm.DSPARK):
|
||||
pools = self._run(
|
||||
algorithm=algorithm,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
)
|
||||
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
|
||||
self.assertIs(pools.token_to_kv_pool_allocator, alloc)
|
||||
|
||||
def test_a_draft_kv_dtype_unlike_the_region_refuses_to_bind(self):
|
||||
"""The region stores the target's KV dtype. A draft that resolved its
|
||||
own -- `auto` picking up an fp8 quant config the target lacks, say --
|
||||
must fail loudly rather than read those rows as another dtype."""
|
||||
alloc = self._swa_allocator(with_draft_region=True)
|
||||
"""The region stores the target's KV dtype, every fp8 flavor as uint8:
|
||||
a draft resolving the target's fp8 binds a pool that casts to it, and
|
||||
one resolving another dtype refuses instead of reading the rows as
|
||||
that dtype."""
|
||||
bf16 = self._swa_allocator(with_draft_region=True)
|
||||
with self.assertRaisesRegex(ValueError, "speculative-draft-kv-cache-dtype"):
|
||||
self._run(
|
||||
algorithm=SpeculativeAlgorithm.EAGLE3,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
alloc=bf16,
|
||||
max_total_num_tokens=bf16.size_full,
|
||||
kv_dtype=torch.float8_e4m3fn,
|
||||
)
|
||||
|
||||
def test_an_fp8_region_binds_an_fp8_draft_and_refuses_another_fp8(self):
|
||||
"""Every fp8 flavor is stored as uint8, so the bind compares KV dtypes:
|
||||
a draft resolving the target's fp8 binds a pool that casts to that fp8,
|
||||
and one resolving a different fp8 refuses instead of reading e4m3 bytes
|
||||
as e5m2."""
|
||||
alloc = self._swa_allocator(
|
||||
with_draft_region=True, kv_dtype=torch.float8_e4m3fn
|
||||
)
|
||||
@@ -372,30 +287,6 @@ class TestDraftBindingDispatch(CustomTestCase):
|
||||
kv_dtype=torch.float8_e5m2,
|
||||
)
|
||||
|
||||
def test_a_draft_with_state_layers_refuses_to_bind(self):
|
||||
"""The target places a draft from its config; the built model is the
|
||||
ground truth. Recurrent layers would run with no state pool."""
|
||||
alloc = self._swa_allocator(with_draft_region=True)
|
||||
with self.assertRaisesRegex(ValueError, "MambaMixer2"):
|
||||
self._run(
|
||||
algorithm=SpeculativeAlgorithm.EAGLE3,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
state_layers=["MambaMixer2"],
|
||||
)
|
||||
|
||||
def test_dspark_draft_with_a_region_binds_the_fused_pool(self):
|
||||
"""DSPARK's draft KV fuses when the target resolved a region (its
|
||||
block rows are indexed by the same token->page identity); the
|
||||
region-less private arm above remains the automatic fallback."""
|
||||
alloc = self._swa_allocator(with_draft_region=True)
|
||||
pools = self._run(
|
||||
algorithm=SpeculativeAlgorithm.DSPARK,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
)
|
||||
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
|
||||
|
||||
def test_compact_dflash_draft_fuses_with_a_private_req_table(self):
|
||||
"""Compact-window DFLASH passes req_to_token_pool=None (it keeps a
|
||||
private table narrowing WHICH pages the draft reads) while the KV
|
||||
@@ -411,22 +302,6 @@ class TestDraftBindingDispatch(CustomTestCase):
|
||||
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
|
||||
self.assertEqual(pools.req_to_token_pool.kind, "private-compact")
|
||||
|
||||
def test_eagle_draft_without_a_placement_falls_back_to_the_private_arm(self):
|
||||
"""Target boot declines a placement for legitimate configurations (a
|
||||
draft with SWA layers of its own, asymmetric rows), so a
|
||||
placement-less EAGLE draft binds the private pool -- sized by the id
|
||||
space like any other private draft -- instead of failing the boot."""
|
||||
alloc = self._swa_allocator(with_draft_region=False)
|
||||
id_space = alloc.full_attn_allocator.max_slots - 1
|
||||
with self.assertRaises(_CapturedSizes) as caught:
|
||||
self._run(
|
||||
algorithm=SpeculativeAlgorithm.EAGLE3,
|
||||
alloc=alloc,
|
||||
max_total_num_tokens=alloc.size_full,
|
||||
)
|
||||
sized = caught.exception.sizes.max_total_num_tokens
|
||||
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -11,7 +11,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""KVIndexTranslator -- the read-path id translator.
|
||||
"""KVIndexTranslator -- what a runner reads and writes an iteration's plan
|
||||
through: the plan's read table and write ids per disposition, the readers
|
||||
(`read_source`, `pack_read_stream`, `copy_page_table`), and the boot-time
|
||||
dispositions.
|
||||
|
||||
CPU-only: these exercise the builder's pure-torch reference path, not the
|
||||
Triton kernel.
|
||||
@@ -32,7 +35,8 @@ from test_multi_ended_allocator import _FakeKVCache, _FakeUnifiedSWAKVPool
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator, KVReadTables
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
|
||||
from sglang.srt.state_capturer.base import BaseTopkCapturer
|
||||
@@ -113,12 +117,26 @@ def _reference_table(req_to_token, req_pool_indices, seq_lens, v2p, ps, width):
|
||||
return out
|
||||
|
||||
|
||||
def _publish():
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
|
||||
def _plan(src, rows, seq_lens, *, write_virtual=None, read_extent=0):
|
||||
return src.plan(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens.clone(),
|
||||
write_virtual=write_virtual,
|
||||
read_extent=read_extent,
|
||||
)
|
||||
|
||||
|
||||
class TestPassthrough(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
_publish()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def test_non_unified_returns_same_objects(self):
|
||||
"""Strict passthrough: no copy, no branch. Any tensor op on the
|
||||
@@ -134,18 +152,15 @@ class TestPassthrough(unittest.TestCase):
|
||||
self.assertFalse(src.is_translating)
|
||||
self.assertEqual(src.capture_token_capacity(17), 18)
|
||||
rows = torch.tensor([2, 0])
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=rows, seq_lens=torch.tensor([5, 3])
|
||||
)
|
||||
loc = torch.tensor([1, 2, 3])
|
||||
plan = _plan(src, rows, torch.tensor([5, 3]), write_virtual=loc)
|
||||
self.assertIs(plan.write_ids(src), loc)
|
||||
view = src.read_source(plan, req_pool_indices=rows, bs=2)
|
||||
self.assertIs(view.ids, req_to_token)
|
||||
self.assertIs(view.row_ids, rows)
|
||||
self.assertEqual(view.row_stride, req_to_token.stride(0))
|
||||
self.assertEqual(view.entry_page_size, 1)
|
||||
self.assertFalse(view.is_translated)
|
||||
self.assertIsNone(view.sliding_window_ids)
|
||||
# And the translate surface is the identity, not a wrapped copy.
|
||||
t = torch.tensor([1, 2, 3])
|
||||
self.assertIs(src.translate_full_attn_ids(t), t)
|
||||
|
||||
|
||||
def _alloc_and_fill(allocator, ps, lens):
|
||||
@@ -167,10 +182,8 @@ def _alloc_and_fill(allocator, ps, lens):
|
||||
|
||||
class TestReadTableBuild(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
_publish()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def test_read_table_matches_reference(self):
|
||||
"""Both read tables must equal the independent per-element derivation
|
||||
@@ -183,71 +196,27 @@ class TestReadTableBuild(unittest.TestCase):
|
||||
)
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
self.assertTrue(src.is_translating)
|
||||
width = 6
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=rows, seq_lens=seq_lens, max_pages=width
|
||||
)
|
||||
plan = _plan(src, rows, seq_lens)
|
||||
view = src.read_table(plan)
|
||||
width = view.ids.shape[1]
|
||||
self.assertEqual(width, 5) # the widest row's pages
|
||||
self.assertTrue(view.is_translated)
|
||||
self.assertEqual(view.entry_page_size, ps)
|
||||
self.assertTrue(
|
||||
torch.equal(view.row_ids, torch.arange(3, dtype=torch.int64))
|
||||
)
|
||||
want_full = _reference_table(
|
||||
req_to_token,
|
||||
rows,
|
||||
seq_lens,
|
||||
allocator.full_v2p_page_table,
|
||||
ps,
|
||||
width,
|
||||
)
|
||||
want_swa = _reference_table(
|
||||
req_to_token,
|
||||
rows,
|
||||
seq_lens,
|
||||
allocator.swa_v2p_page_table,
|
||||
ps,
|
||||
width,
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(view.ids, want_full),
|
||||
f"full read table off-formula (ps={ps})",
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(view.sliding_window_ids, want_swa),
|
||||
f"swa read table off-formula (ps={ps})",
|
||||
)
|
||||
|
||||
def test_packed_stream_equals_the_rectangle_it_replaces(self):
|
||||
"""The two builders must agree element for element:
|
||||
packed[indptr[b] + p] == ids[b, p // ps] * ps + p % ps."""
|
||||
for ps in (1, 4):
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(
|
||||
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
|
||||
)
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=rows, seq_lens=seq_lens, max_pages=6
|
||||
)
|
||||
indptr = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
|
||||
indptr[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
total = int(indptr[-1])
|
||||
packed = torch.zeros(total, dtype=torch.int32)
|
||||
translated = src.fill_packed_read_stream(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=seq_lens,
|
||||
indptr=indptr,
|
||||
total_tokens=total,
|
||||
out=packed,
|
||||
)
|
||||
self.assertTrue(translated)
|
||||
for b, n in enumerate(seq_lens.tolist()):
|
||||
for pos in range(n):
|
||||
self.assertEqual(
|
||||
int(packed[int(indptr[b]) + pos]),
|
||||
int(view.ids[b, pos // ps]) * ps + pos % ps,
|
||||
f"packed stream off the page table (ps={ps}, b={b}, {pos=})",
|
||||
)
|
||||
for table, v2p, side in (
|
||||
(view.ids, allocator.full_v2p_page_table, "full"),
|
||||
(
|
||||
src.read_table(plan, kind=IdSpaceKind.SLIDING_WINDOW).ids,
|
||||
allocator.swa_v2p_page_table,
|
||||
"swa",
|
||||
),
|
||||
):
|
||||
want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width)
|
||||
self.assertTrue(
|
||||
torch.equal(table, want), f"{side} read table off-formula (ps={ps})"
|
||||
)
|
||||
|
||||
def test_sink_routing(self):
|
||||
"""Dead lanes, -1 slots inside the live prefix, and tombstoned v2p
|
||||
@@ -265,64 +234,112 @@ class TestReadTableBuild(unittest.TestCase):
|
||||
allocator.full_v2p_page_table[tomb_page] = -1
|
||||
allocator.swa_v2p_page_table[tomb_page] = -1
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=rows, seq_lens=seq_lens, max_pages=4
|
||||
)
|
||||
for table in (view.ids, view.sliding_window_ids):
|
||||
plan = _plan(src, rows, seq_lens)
|
||||
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
|
||||
table = src.read_table(plan, kind=kind).ids
|
||||
self.assertTrue(bool((table >= 0).all()))
|
||||
self.assertTrue(bool((table[1] == 0).all()), "dead lane not sunk")
|
||||
self.assertEqual(int(table[0, 1]), 0, "-1 slot not sunk")
|
||||
self.assertEqual(int(table[2, 0]), 0, "tombstone not sunk")
|
||||
|
||||
|
||||
class TestBuildInto(unittest.TestCase):
|
||||
"""fill_read_table fills a backend-owned padded block table's live prefix with
|
||||
FULL-side read-table entries -- the trtllm_mla / flashmla consumption route
|
||||
(their rows ARE the read table's rows)."""
|
||||
class TestReadExtent(unittest.TestCase):
|
||||
"""A forward that reads past its lengths (a verify's draft tail) plans its
|
||||
table that far: `seq_lens + read_extent`, capped at the request row."""
|
||||
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
_publish()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def test_prefix_filled_tail_sentinel_preserved_width_capped(self):
|
||||
"""The -1 tail sentinel belongs to the backend, and a table padded
|
||||
WIDER than the req_to_token page span (trtllm's LCM alignment) must be
|
||||
capped rather than trip the builder's width assert."""
|
||||
def test_table_covers_the_extent(self):
|
||||
window = 3
|
||||
for ps in (1, 4):
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token, rows, live = _alloc_and_fill(
|
||||
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
|
||||
)
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
view = src.read_table(_plan(src, rows, live, read_extent=window))
|
||||
width = -(-(int(live.max()) + window) // ps)
|
||||
self.assertEqual(tuple(view.ids.shape), (rows.numel(), width))
|
||||
want = _reference_table(
|
||||
req_to_token,
|
||||
rows,
|
||||
live + window,
|
||||
allocator.full_v2p_page_table,
|
||||
ps,
|
||||
width,
|
||||
)
|
||||
self.assertTrue(torch.equal(view.ids, want), f"ps={ps}")
|
||||
|
||||
def test_width_never_exceeds_the_request_row(self):
|
||||
"""Near the context limit the host bound plus the extent can pass the
|
||||
end of the `req_to_token` row; the table stops at the row instead of
|
||||
tripping the read-table width assert."""
|
||||
window = 6
|
||||
allocator = _build_composite(1)
|
||||
req_to_token, rows, live = _alloc_and_fill(allocator, 1, lens=[6, 1])
|
||||
row_pages = req_to_token.shape[1]
|
||||
src = _make_source(allocator, req_to_token, 1)
|
||||
plan = src.plan(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=live,
|
||||
seq_lens_cpu=live + 2 * window,
|
||||
write_virtual=None,
|
||||
read_extent=window,
|
||||
)
|
||||
self.assertGreater(int(live.max()) + 3 * window, row_pages)
|
||||
self.assertEqual(src.read_table(plan).ids.shape[1], row_pages)
|
||||
|
||||
|
||||
class TestCopyPageTable(unittest.TestCase):
|
||||
"""`copy_page_table`: a captured graph's own table takes the plan's rows,
|
||||
padded lanes reading the sink; columns past the plan's width keep their
|
||||
values (the kernels bound their reads by their own lengths)."""
|
||||
|
||||
def setUp(self):
|
||||
_publish()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def test_rows_copied_tail_kept_padded_lanes_sunk(self):
|
||||
ps = 4
|
||||
allocator = _build_composite(ps)
|
||||
lens = [5, 2 * ps + 1, 1]
|
||||
lens = [5, 2 * ps + 1]
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens)
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
self.assertTrue(src.is_translating)
|
||||
plan = _plan(src, rows, seq_lens)
|
||||
width = src.read_table(plan).ids.shape[1]
|
||||
# Wider than the req_to_token span and one lane more than the batch.
|
||||
out = torch.full((3, req_to_token.shape[1] // ps + 3), 7, dtype=torch.int32)
|
||||
out_swa = torch.full_like(out, 7)
|
||||
src.copy_page_table(plan, out=out)
|
||||
src.copy_page_table(plan, out=out_swa, kind=IdSpaceKind.SLIDING_WINDOW)
|
||||
for v2p, got in (
|
||||
(allocator.full_v2p_page_table, out),
|
||||
(allocator.swa_v2p_page_table, out_swa),
|
||||
):
|
||||
want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width)
|
||||
self.assertTrue(torch.equal(got[:2, :width], want))
|
||||
self.assertTrue(bool((got[2, :width] == 0).all()), "padded lane")
|
||||
self.assertTrue(bool((got[:, width:] == 7).all()), "tail clobbered")
|
||||
|
||||
width_pages = req_to_token.shape[1] // ps + 3 # wider than the span
|
||||
out = torch.full((len(lens), width_pages), -1, dtype=torch.int32)
|
||||
src.fill_read_table(out=out, req_pool_indices=rows, seq_lens=seq_lens)
|
||||
|
||||
want = _reference_table(
|
||||
req_to_token,
|
||||
rows,
|
||||
seq_lens,
|
||||
allocator.full_v2p_page_table,
|
||||
ps,
|
||||
width_pages,
|
||||
)
|
||||
for b, n in enumerate(lens):
|
||||
n_pages = -(-n // ps)
|
||||
self.assertTrue(
|
||||
torch.equal(out[b, :n_pages], want[b, :n_pages]),
|
||||
f"row {b} live prefix off-formula",
|
||||
)
|
||||
self.assertTrue(
|
||||
bool((out[b, n_pages:] == -1).all()),
|
||||
f"row {b} tail sentinel clobbered",
|
||||
)
|
||||
def test_row_ids_not_reallocated_across_plans(self):
|
||||
"""`row_ids` is one arange sized from the request pool and sliced per
|
||||
table; a per-table `torch.arange` would cost an allocation and a launch
|
||||
on every replay prep."""
|
||||
allocator = _build_composite(1)
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3])
|
||||
src = _make_source(allocator, req_to_token, 1)
|
||||
first = src.read_table(_plan(src, rows[:2], seq_lens[:2]))
|
||||
second = src.read_table(_plan(src, rows, seq_lens))
|
||||
self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr())
|
||||
self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV)))
|
||||
self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV)))
|
||||
self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0])
|
||||
|
||||
def test_passthrough_source_refuses(self):
|
||||
"""Callers dispatch on `reads_are_translated`; a passthrough source has
|
||||
no v2p to build from and must fail loud, not fill garbage."""
|
||||
no plan table to copy and must fail loud, not fill garbage."""
|
||||
src = KVIndexTranslator(
|
||||
req_to_token=torch.zeros((2, 4), dtype=torch.int64),
|
||||
token_to_kv_pool_allocator=SimpleNamespace(),
|
||||
@@ -330,12 +347,9 @@ class TestBuildInto(unittest.TestCase):
|
||||
page_size=1,
|
||||
device=_DEV,
|
||||
)
|
||||
plan = _plan(src, torch.tensor([0]), torch.tensor([1]))
|
||||
with self.assertRaises(AssertionError):
|
||||
src.fill_read_table(
|
||||
out=torch.zeros((1, 4), dtype=torch.int32),
|
||||
req_pool_indices=torch.tensor([0]),
|
||||
seq_lens=torch.tensor([1]),
|
||||
)
|
||||
src.copy_page_table(plan, out=torch.zeros((1, 4), dtype=torch.int32))
|
||||
|
||||
|
||||
class TestPoolOwnership(unittest.TestCase):
|
||||
@@ -422,81 +436,26 @@ class TestPoolOwnership(unittest.TestCase):
|
||||
device=_DEV,
|
||||
)
|
||||
rows = torch.tensor([1, 0], dtype=torch.int32, device=_DEV)
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=torch.tensor([3, 2], dtype=torch.int32, device=_DEV),
|
||||
loc = torch.tensor([5, 6], dtype=torch.int64, device=_DEV)
|
||||
plan = _plan(
|
||||
src,
|
||||
rows,
|
||||
torch.tensor([3, 2], dtype=torch.int32, device=_DEV),
|
||||
write_virtual=loc,
|
||||
)
|
||||
# Read table: the EXACT objects a static-pool backend reads today.
|
||||
# Writes and reads: the EXACT objects a static-pool backend uses.
|
||||
self.assertIs(plan.write_ids(src), loc)
|
||||
view = src.read_source(plan, req_pool_indices=rows, bs=2)
|
||||
self.assertIs(view.ids, req_to_token)
|
||||
self.assertIs(view.row_ids, rows)
|
||||
self.assertFalse(view.is_translated)
|
||||
# And the token-level surface is the identity, same guard.
|
||||
t = torch.tensor([5, 6], dtype=torch.int64, device=_DEV)
|
||||
self.assertIs(src.translate_full_attn_ids(t), t)
|
||||
|
||||
|
||||
class TestCaptureContract(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def test_caller_owned_table_is_returned_whole_and_filled_prefix_only(self):
|
||||
ps = 4
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token = torch.full((4, 16 * ps), -1, dtype=torch.int64)
|
||||
v = allocator.alloc(2 * ps)
|
||||
req_to_token[1, : 2 * ps] = v
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
|
||||
# A page-table consumer owns its buffers; entry 0 is the reserved sink
|
||||
# in every id space, so zeros are what an unfilled column must read as.
|
||||
cap = torch.zeros((4, 8), dtype=torch.int32, device=_DEV)
|
||||
cap_swa = torch.zeros((4, 8), dtype=torch.int32, device=_DEV)
|
||||
tables = KVReadTables(full=cap, sliding_window=cap_swa)
|
||||
|
||||
# Poison everything, then refresh a 1-row batch: ONLY its live prefix
|
||||
# may change -- stale tails and other rows are the fa3 contract.
|
||||
cap.fill_(7)
|
||||
cap_swa.fill_(7)
|
||||
view = src.build_index_table(
|
||||
req_pool_indices=torch.tensor([1]),
|
||||
seq_lens=torch.tensor([2 * ps]),
|
||||
into=tables,
|
||||
)
|
||||
self.assertIs(view.ids, cap, "the caller's table comes back WHOLE")
|
||||
want = allocator.full_v2p_page_table[req_to_token[1, ::ps][:2] // ps]
|
||||
self.assertTrue(torch.equal(cap[0, :2], want.to(torch.int32)))
|
||||
self.assertTrue(bool((cap[0, 2:] == 7).all()), "stale tail was cleared")
|
||||
self.assertTrue(bool((cap[1:] == 7).all()), "rows beyond bs were touched")
|
||||
|
||||
def test_row_ids_not_reallocated_across_builds(self):
|
||||
"""`row_ids` is one arange sized from the request pool and sliced per
|
||||
build; a per-build `torch.arange` would be correct but costs an
|
||||
allocation and a launch on every replay prep."""
|
||||
allocator = _build_composite(1)
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3])
|
||||
src = _make_source(allocator, req_to_token, 1)
|
||||
self.assertTrue(src.is_translating)
|
||||
first = src.build_index_table(
|
||||
req_pool_indices=rows[:2], seq_lens=seq_lens[:2], max_pages=4
|
||||
)
|
||||
second = src.build_index_table(
|
||||
req_pool_indices=rows, seq_lens=seq_lens, max_pages=4
|
||||
)
|
||||
self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr())
|
||||
self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV)))
|
||||
self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV)))
|
||||
# Sized to bound any batch the request pool can hold.
|
||||
self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0])
|
||||
|
||||
|
||||
class _FakeForwardBatch:
|
||||
"""Weakref-able stand-in (SimpleNamespace is not) carrying the fields
|
||||
`index_table_for_batch` and `rebind_write_loc` read. `seq_lens_sum`
|
||||
defaults to the real sum: it is the signal that the CPU mirror is live,
|
||||
and a real ForwardBatch always carries it (None only when gpu_only)."""
|
||||
"""Stand-in carrying the fields a plan of its own (`own_plan`) and the
|
||||
capturers read. `seq_lens_sum` defaults to the real sum: it is the signal
|
||||
that the CPU mirror is live, and a real ForwardBatch always carries it
|
||||
(None only when gpu_only)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -520,138 +479,14 @@ class _FakeForwardBatch:
|
||||
)
|
||||
|
||||
|
||||
class TestWidenedTable(unittest.TestCase):
|
||||
"""`widened_index_table`: the whole-sequence verify view, `seq_len_delta`
|
||||
columns past every row's live prefix."""
|
||||
class TestWriteIds(CustomTestCase):
|
||||
"""A plan translates its write window once, into a fresh tensor, and
|
||||
derives the sliding-window ids from the virtual window -- pads and
|
||||
tombstones landing on the sink."""
|
||||
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
_publish()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def _check(self, view, req_to_token, rows, lens, allocator, ps, width):
|
||||
self.assertEqual(tuple(view.ids.shape), (rows.numel(), width))
|
||||
want = _reference_table(
|
||||
req_to_token, rows, lens, allocator.full_v2p_page_table, ps, width
|
||||
)
|
||||
self.assertTrue(torch.equal(view.ids, want), f"ps={ps}")
|
||||
|
||||
def test_width_comes_from_the_live_prefix(self):
|
||||
"""A DSPARK verify batch carries `seq_lens_cpu` already expanded by the
|
||||
window and the live lens on `spec_info.live_seq_lens_cpu`; widening the
|
||||
expanded lens would count the window twice."""
|
||||
window = 3
|
||||
for ps in (1, 4):
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token, rows, live = _alloc_and_fill(
|
||||
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
|
||||
)
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
fb = _FakeForwardBatch(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=live,
|
||||
seq_lens_cpu=live + window,
|
||||
spec_info=SimpleNamespace(live_seq_lens_cpu=live),
|
||||
)
|
||||
view = src.widened_index_table(fb, seq_len_delta=window)
|
||||
width = -(-(int(live.max()) + window) // ps)
|
||||
self._check(view, req_to_token, rows, live + window, allocator, ps, width)
|
||||
|
||||
def test_width_never_exceeds_the_request_row(self):
|
||||
"""Near the context limit the widened host bound can pass the end of
|
||||
the `req_to_token` row; the table stops at the row instead of tripping
|
||||
the read-table width assert."""
|
||||
window = 6
|
||||
ps = 1
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token, rows, live = _alloc_and_fill(allocator, ps, lens=[6, 1])
|
||||
row_pages = req_to_token.shape[1] // ps
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
# No live lens on the spec info: the expanded mirror is all there is.
|
||||
fb = _FakeForwardBatch(
|
||||
req_pool_indices=rows, seq_lens=live, seq_lens_cpu=live + window
|
||||
)
|
||||
self.assertGreater(int(live.max()) + 2 * window, row_pages)
|
||||
view = src.widened_index_table(fb, seq_len_delta=window)
|
||||
self._check(view, req_to_token, rows, live + window, allocator, ps, row_pages)
|
||||
|
||||
|
||||
class TestViewMemo(unittest.TestCase):
|
||||
"""The eager view is memoized ON THE SOURCE in a single slot keyed by batch
|
||||
identity, so per-batch state stays out of the ForwardBatch while one
|
||||
metadata build's many consumers still share one table build."""
|
||||
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def _fb(self, allocator, ps, lens):
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens)
|
||||
fb = _FakeForwardBatch(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
)
|
||||
return fb, req_to_token
|
||||
|
||||
def test_same_batch_returns_the_memoized_view(self):
|
||||
ps = 1
|
||||
allocator = _build_composite(ps)
|
||||
fb, req_to_token = self._fb(allocator, ps, lens=[3, 2])
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
v1 = src.index_table_for_batch(fb)
|
||||
v2 = src.index_table_for_batch(fb)
|
||||
self.assertIs(v1, v2)
|
||||
|
||||
def test_next_batch_replaces_the_single_slot(self):
|
||||
ps = 1
|
||||
allocator = _build_composite(ps)
|
||||
fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2])
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
v1 = src.index_table_for_batch(fb1)
|
||||
fb2 = _FakeForwardBatch(
|
||||
req_pool_indices=fb1.req_pool_indices,
|
||||
seq_lens=fb1.seq_lens,
|
||||
seq_lens_cpu=fb1.seq_lens_cpu,
|
||||
)
|
||||
v2 = src.index_table_for_batch(fb2)
|
||||
self.assertIsNot(v1, v2)
|
||||
# Single slot: fb1 no longer matches and rebuilds.
|
||||
v1_again = src.index_table_for_batch(fb1)
|
||||
self.assertIsNot(v1_again, v1)
|
||||
|
||||
def test_dead_batch_never_matches(self):
|
||||
"""A garbage-collected batch's slot must not serve a later batch: the
|
||||
weakref key goes dead and the build runs fresh."""
|
||||
import gc
|
||||
|
||||
ps = 1
|
||||
allocator = _build_composite(ps)
|
||||
fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2])
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
v1 = src.index_table_for_batch(fb1)
|
||||
del fb1
|
||||
gc.collect()
|
||||
fb2, _ = self._fb(allocator, ps, lens=[2])
|
||||
v2 = src.index_table_for_batch(fb2)
|
||||
self.assertIsNot(v2, v1)
|
||||
self.assertEqual(v2.ids.shape[0], 1)
|
||||
|
||||
|
||||
class TestWriteLoc(CustomTestCase):
|
||||
"""The two-phase write contract: `rebind_write_loc` rebinds the full side
|
||||
once at ForwardBatch construction, and the sliding-window write loc derives
|
||||
POINTWISE from the full-side values -- pads, slices, and fresh copies
|
||||
included -- with no handover and no stored per-forward state."""
|
||||
|
||||
def setUp(self):
|
||||
# The code under test reads its config from the bags.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def _built(self, ps=1, n=4):
|
||||
allocator = _build_composite(ps)
|
||||
@@ -662,20 +497,18 @@ class TestWriteLoc(CustomTestCase):
|
||||
want_swa = allocator.translate_loc_from_full_to_swa(virt)
|
||||
return src, allocator, rows, seq_lens, virt, want_full, want_swa
|
||||
|
||||
def _field(self, src, rows, seq_lens, kernel_loc):
|
||||
return src.sliding_window_write_loc_for(kernel_loc)
|
||||
|
||||
def test_rebind_translates_full_side_only(self):
|
||||
def test_own_plan_translates_full_side_only(self):
|
||||
for ps in (1, 4):
|
||||
src, _, _, _, virt, want_full, _ = self._built(ps=ps, n=3 * ps)
|
||||
keep = virt.clone()
|
||||
fb = _FakeForwardBatch(out_cache_loc=virt)
|
||||
src.rebind_write_loc(fb)
|
||||
# Full side: rebound to a FRESH physical tensor; the
|
||||
# ScheduleBatch's aliased virtual tensor is untouched.
|
||||
src.bind_own_plan(fb)
|
||||
# Full side: a FRESH physical tensor; the ScheduleBatch's aliased
|
||||
# virtual tensor is untouched and mirrored for its bookkeeping.
|
||||
self.assertIsNot(fb.out_cache_loc, virt)
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, want_full))
|
||||
self.assertTrue(torch.equal(virt, keep))
|
||||
self.assertIs(fb.out_cache_loc_virtual, virt)
|
||||
|
||||
def test_capture_capacity_covers_dcp_widened_mamba_ids(self):
|
||||
"""A capturer must store the highest virtual IDs issued under DCP."""
|
||||
@@ -755,7 +588,7 @@ class TestWriteLoc(CustomTestCase):
|
||||
req_pool = SimpleNamespace(req_to_token=virt.clone()[None, :])
|
||||
fb = _FakeForwardBatch(out_cache_loc=virt.clone())
|
||||
if translating:
|
||||
src.rebind_write_loc(fb)
|
||||
src.bind_own_plan(fb)
|
||||
fb.out_cache_loc = torch.cat(
|
||||
[fb.out_cache_loc, virt.new_zeros(2)]
|
||||
)
|
||||
@@ -795,35 +628,36 @@ class TestWriteLoc(CustomTestCase):
|
||||
)
|
||||
)
|
||||
|
||||
def test_swa_write_loc_round_trips_from_full_side(self):
|
||||
"""Derived property: `field(full(t)) == swa(t)` for any virtual run t,
|
||||
across page sizes."""
|
||||
def test_swa_ids_come_from_the_virtual_ids(self):
|
||||
for ps in (1, 4, 64):
|
||||
src, _, rows, seq_lens, _, want_full, want_swa = self._built(
|
||||
ps=ps, n=3 * ps
|
||||
src, _, rows, seq_lens, virt, _, want_swa = self._built(ps=ps, n=3 * ps)
|
||||
plan = _plan(src, rows, seq_lens, write_virtual=virt)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW), want_swa
|
||||
)
|
||||
)
|
||||
got = self._field(src, rows, seq_lens, want_full)
|
||||
self.assertTrue(torch.equal(got, want_swa))
|
||||
|
||||
def test_pad_lanes_derive_to_sink(self):
|
||||
"""The DP pad appends zeros, and physical 0 is the reserved
|
||||
padding slot in every id space, so pad lanes derive to swa slot 0 with
|
||||
no `num_live` bookkeeping."""
|
||||
src, _, rows, seq_lens, _, want_full, want_swa = self._built(n=3)
|
||||
padded = torch.cat([want_full, want_full.new_zeros(2)])
|
||||
got = self._field(src, rows, seq_lens, padded)
|
||||
def test_pad_lanes_read_the_sink(self):
|
||||
"""The DP pad appends zeros, and id 0 is the reserved padding slot in
|
||||
every id space, so pad lanes land on swa slot 0."""
|
||||
src, _, rows, seq_lens, virt, _, want_swa = self._built(n=3)
|
||||
padded = torch.cat([virt, virt.new_zeros(2)])
|
||||
plan = _plan(src, rows, seq_lens, write_virtual=padded)
|
||||
got = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
|
||||
self.assertTrue(torch.equal(got[:3], want_swa))
|
||||
self.assertTrue(bool((got[3:] == 0).all()), "pad lanes must land on slot 0")
|
||||
|
||||
def test_tombstoned_swa_page_clamps_to_sink(self):
|
||||
src, allocator, rows, seq_lens, virt, want_full, _ = self._built(ps=1, n=2)
|
||||
def test_tombstoned_swa_page_reads_the_sink(self):
|
||||
src, allocator, rows, seq_lens, virt, _, _ = self._built(ps=1, n=2)
|
||||
allocator.swa_v2p_page_table[int(virt[0])] = -1
|
||||
got = self._field(src, rows, seq_lens, want_full[:1])
|
||||
self.assertEqual(int(got[0]), 0)
|
||||
plan = _plan(src, rows, seq_lens, write_virtual=virt[:1])
|
||||
swa_ids = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
|
||||
self.assertEqual(int(swa_ids[0]), 0)
|
||||
|
||||
def test_static_swa_pool_derives_via_pool_translate(self):
|
||||
"""Static SWA pools: the field is the pool's own legacy full->swa
|
||||
translate, computed at the same build; the rebind stays a no-op."""
|
||||
"""Static SWA pools: the swa ids are the pool's own full->swa
|
||||
translate; the write ids stay the batch's own."""
|
||||
pool = SWAKVPool.__new__(SWAKVPool)
|
||||
pool.full_to_swa_index_mapping = torch.arange(10, dtype=torch.int64)
|
||||
pool.translate_loc_from_full_to_swa = lambda t: t + 100
|
||||
@@ -836,14 +670,18 @@ class TestWriteLoc(CustomTestCase):
|
||||
)
|
||||
loc = torch.tensor([5, 6], dtype=torch.int64)
|
||||
fb = _FakeForwardBatch(out_cache_loc=loc)
|
||||
src.rebind_write_loc(fb)
|
||||
self.assertIs(fb.out_cache_loc, loc, "disabled rebind must be a no-op")
|
||||
self.assertTrue(torch.equal(src.sliding_window_write_loc_for(loc), loc + 100))
|
||||
src.bind_own_plan(fb)
|
||||
self.assertIs(fb.out_cache_loc, loc, "a static pool's ids are its own")
|
||||
self.assertTrue(
|
||||
torch.equal(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), loc + 100)
|
||||
)
|
||||
|
||||
def test_no_loc_or_no_swa_side_yields_none(self):
|
||||
# Unified swa composite, but there is no write loc this forward.
|
||||
src, _, rows, seq_lens, _, _, _ = self._built(n=2)
|
||||
self.assertIsNone(src.sliding_window_write_loc_for(None))
|
||||
self.assertIsNone(
|
||||
_plan(src, rows, seq_lens).write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
|
||||
)
|
||||
# Passthrough on a non-SWA pool: a loc is given, but there is no swa
|
||||
# id space to derive into.
|
||||
plain = KVIndexTranslator(
|
||||
@@ -853,23 +691,13 @@ class TestWriteLoc(CustomTestCase):
|
||||
page_size=1,
|
||||
device=_DEV,
|
||||
)
|
||||
loc = torch.tensor([3], dtype=torch.int64)
|
||||
self.assertIsNone(
|
||||
plain.sliding_window_write_loc_for(torch.tensor([3], dtype=torch.int64))
|
||||
_plan(
|
||||
plain, torch.tensor([0]), torch.tensor([1]), write_virtual=loc
|
||||
).write_ids(plain, kind=IdSpaceKind.SLIDING_WINDOW)
|
||||
)
|
||||
|
||||
def test_rebind_retires_the_view_memo(self):
|
||||
ps = 1
|
||||
allocator = _build_composite(ps)
|
||||
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=[3, 2])
|
||||
src = _make_source(allocator, req_to_token, ps)
|
||||
fb = _FakeForwardBatch(
|
||||
req_pool_indices=rows, seq_lens=seq_lens, seq_lens_cpu=seq_lens
|
||||
)
|
||||
v1 = src.index_table_for_batch(fb)
|
||||
src.rebind_write_loc(_FakeForwardBatch(out_cache_loc=None))
|
||||
v2 = src.index_table_for_batch(fb)
|
||||
self.assertIsNot(v2, v1, "rebind starts the next forward: stale views die")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -11,36 +11,29 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""KVIndexTranslator's three id-space dispositions with a fused draft region.
|
||||
"""KVIndexTranslator with a fused draft region.
|
||||
|
||||
1. TARGET runner (allocator's own kvcache): translates to physical ids.
|
||||
2. FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to this allocator): the
|
||||
SAME translate -- same pages, same v2p table, same physical ids (the draft
|
||||
parts sit inside the host entry). A dense draft pool routes no window
|
||||
layers, so it has no swa id space: the window index table falls back to
|
||||
the one dense table and `sliding_window_write_loc_for()` answers None (a
|
||||
separate-swa assumption here crashed the read side and left the write side
|
||||
with nothing to derive).
|
||||
3. PRIVATE-POOL draft (DSPARK/DFLASH shape: target's allocator, own
|
||||
virtual-indexed buffer): strict passthrough — translating such a runner
|
||||
would address a slot-count buffer with dense ids (OOB both directions),
|
||||
so the no-op is load-bearing, not a default.
|
||||
A FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to the target's allocator)
|
||||
translates exactly as the target does -- same v2p table, same physical ids,
|
||||
the draft parts sitting inside the host entries -- and, routing no window
|
||||
layers, has no sliding-window write ids. A PRIVATE-POOL draft (own
|
||||
virtual-indexed buffer) stays a strict passthrough. The draft writers outside
|
||||
the draft's forward (DFLASH/DSPARK's target-hidden writes, the multi-step
|
||||
draft decode's kv indices) read the same ids.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
|
||||
"""
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache import kv_index_translator
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
DenseDraftRegion,
|
||||
FusedDraftPlacement,
|
||||
@@ -51,7 +44,7 @@ from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
|
||||
|
||||
_DEV = "cpu"
|
||||
_PS = 2
|
||||
@@ -150,163 +143,32 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def test_target_runner_translates_to_physical_ids(self):
|
||||
_, allocator, kvcache, _ = _build()
|
||||
src = _source(allocator, kvcache)
|
||||
self.assertTrue(src.is_translating)
|
||||
v = allocator.alloc(2 * _PS)
|
||||
self.assertIsNotNone(v)
|
||||
fb = SimpleNamespace(out_cache_loc=v)
|
||||
src.rebind_write_loc(fb)
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
|
||||
|
||||
def test_fused_draft_runner_translates_to_the_same_physical_ids(self):
|
||||
_, allocator, _, draft_pool = _build()
|
||||
src = _source(allocator, draft_pool)
|
||||
self.assertTrue(src.is_translating)
|
||||
self.assertIsNone(src._swa_v2p_table) # single space: no swa id space
|
||||
# A dense draft pool: no sliding-window sub-pool.
|
||||
self.assertIsNone(src.space(IdSpaceKind.SLIDING_WINDOW))
|
||||
|
||||
# rebind_write_loc translates to the HOST's physical ids: the draft
|
||||
# parts live inside the same slots.
|
||||
# Its own plan translates to the HOST's physical ids: the draft parts
|
||||
# live inside the same slots.
|
||||
v = allocator.alloc(2 * _PS)
|
||||
self.assertIsNotNone(v)
|
||||
fb = SimpleNamespace(out_cache_loc=v)
|
||||
src.rebind_write_loc(fb)
|
||||
src.bind_own_plan(fb)
|
||||
self.assertIsNot(fb.out_cache_loc, v)
|
||||
fa = allocator.full_attn_allocator
|
||||
expected = torch.clamp_min(fa.virtual_to_physical[v // _PS] * _PS + v % _PS, 0)
|
||||
torch.testing.assert_close(fb.out_cache_loc, expected, rtol=0, atol=0)
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
|
||||
# A dense draft pool routes no window layers: no swa write loc.
|
||||
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
|
||||
|
||||
def test_dense_fused_draft_has_no_window_write_loc(self):
|
||||
"""`sliding_window_write_loc_for()` on a dense fused-draft batch answers
|
||||
None: the pool routes no window layers, so there is nothing to
|
||||
derive. The hybrid-SWA TARGET on the same allocator keeps a genuinely
|
||||
derived swa loc, so the None can never paper over a real swa id
|
||||
space."""
|
||||
_, allocator, kvcache, draft_pool = _build()
|
||||
src = _source(allocator, draft_pool)
|
||||
v = allocator.alloc(_PS)
|
||||
self.assertIsNotNone(v)
|
||||
fb = SimpleNamespace(out_cache_loc=v)
|
||||
src.rebind_write_loc(fb)
|
||||
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
|
||||
# Target contrast: a real swa side derives a DIFFERENT loc.
|
||||
tgt = _source(allocator, kvcache)
|
||||
tv = allocator.alloc(_PS)
|
||||
self.assertIsNotNone(tv)
|
||||
tfb = SimpleNamespace(out_cache_loc=tv)
|
||||
tgt.rebind_write_loc(tfb)
|
||||
tswa = tgt.sliding_window_write_loc_for(tfb.out_cache_loc)
|
||||
self.assertIsNotNone(tswa)
|
||||
self.assertIsNot(tswa, tfb.out_cache_loc)
|
||||
self.assertTrue(torch.equal(tswa, allocator.translate_loc_from_full_to_swa(tv)))
|
||||
|
||||
def test_private_pool_draft_stays_a_strict_passthrough(self):
|
||||
_, allocator, _, _ = _build()
|
||||
private_draft_pool = _FakeKVCache(64) # own buffer, not the allocator's
|
||||
src = _source(allocator, private_draft_pool)
|
||||
self.assertFalse(src.is_translating)
|
||||
v = torch.arange(2 * _PS, dtype=torch.int64)
|
||||
fb = SimpleNamespace(out_cache_loc=v)
|
||||
src.rebind_write_loc(fb)
|
||||
self.assertIs(fb.out_cache_loc, v) # untouched, not even a copy
|
||||
|
||||
def test_foreign_allocator_draft_pool_is_not_enabled(self):
|
||||
# A UnifiedDraftKVPool bound to a DIFFERENT allocator must not enable
|
||||
# against this one (identity, not type, decides).
|
||||
pool, allocator, kvcache, _ = _build()
|
||||
_, other_alloc, _, other_draft = _build()
|
||||
src = _source(allocator, other_draft)
|
||||
self.assertFalse(src.is_translating)
|
||||
|
||||
def test_seq_len_delta_matches_widened_lens(self):
|
||||
"""``seq_len_delta=k`` must be byte-identical to building with
|
||||
``seq_lens + k`` -- the two spellings of the whole-sequence verify
|
||||
widening. A kernel applying the delta to the page count but not the
|
||||
loads (or vice versa) silently truncates the verify tail."""
|
||||
_, allocator, kvcache, _ = _build()
|
||||
v = allocator.alloc(6 * _PS)
|
||||
self.assertIsNotNone(v)
|
||||
rt = torch.zeros((2, 16), dtype=torch.int32)
|
||||
rt[0, : v.numel()] = v.to(torch.int32)
|
||||
src = KVIndexTranslator(
|
||||
req_to_token=rt,
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
token_to_kv_pool=kvcache,
|
||||
page_size=_PS,
|
||||
device=_DEV,
|
||||
)
|
||||
rpi = torch.tensor([0], dtype=torch.int64)
|
||||
seq = torch.tensor([3], dtype=torch.int64)
|
||||
delta = 2 * _PS + 1
|
||||
max_pages = -(-(3 + delta) // _PS)
|
||||
widened = src.build_index_table(
|
||||
req_pool_indices=rpi,
|
||||
seq_lens=seq,
|
||||
max_pages=max_pages,
|
||||
seq_len_delta=delta,
|
||||
)
|
||||
by_lens = src.build_index_table(
|
||||
req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages
|
||||
)
|
||||
torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0)
|
||||
# The delta genuinely widened: entries exist past the unwidened prefix.
|
||||
plain_pages = -(-3 // _PS)
|
||||
self.assertTrue(bool((widened.ids[0, plain_pages:] > 0).any()))
|
||||
|
||||
def test_widened_index_table_matches_widened_lens(self):
|
||||
"""`widened_index_table` (the verify entry point) must equal the
|
||||
widened-lens build over the SAME batch: derive max_pages from the
|
||||
widened max and forward the delta. Deriving max_pages from the
|
||||
un-widened lens silently truncates the verify tail's pages."""
|
||||
_, allocator, kvcache, _ = _build()
|
||||
v = allocator.alloc(6 * _PS)
|
||||
self.assertIsNotNone(v)
|
||||
rt = torch.zeros((2, 16), dtype=torch.int32)
|
||||
rt[0, : v.numel()] = v.to(torch.int32)
|
||||
src = KVIndexTranslator(
|
||||
req_to_token=rt,
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
token_to_kv_pool=kvcache,
|
||||
page_size=_PS,
|
||||
device=_DEV,
|
||||
)
|
||||
rpi = torch.tensor([0], dtype=torch.int64)
|
||||
seq = torch.tensor([3], dtype=torch.int64)
|
||||
delta = 2 * _PS + 1
|
||||
fb = SimpleNamespace(
|
||||
req_pool_indices=rpi,
|
||||
seq_lens=seq,
|
||||
seq_lens_cpu=seq.cpu(),
|
||||
# `seq_lens_sum` is the liveness signal for seq_lens_cpu: it is a
|
||||
# non-None but STALE slice on a gpu_only batch, so the build only
|
||||
# trusts it when the sum is present. Without the field the stand-in
|
||||
# batch falls back to the full req_to_token width.
|
||||
seq_lens_sum=int(seq.sum()),
|
||||
out_cache_loc=None,
|
||||
spec_info=None,
|
||||
)
|
||||
widened = src.widened_index_table(fb, seq_len_delta=delta)
|
||||
max_pages = -(-(3 + delta) // _PS)
|
||||
by_lens = src.build_index_table(
|
||||
req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages
|
||||
)
|
||||
self.assertEqual(widened.ids.shape, by_lens.ids.shape)
|
||||
torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0)
|
||||
self.assertIsNone(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW))
|
||||
|
||||
def test_target_hidden_writers_write_physical_ids(self):
|
||||
"""BUG REGRESSION. DFLASH and DSPARK do not compute their draft KV from
|
||||
the draft's own forward -- they PROJECT the target's hidden states and
|
||||
write them straight into the draft pool, at locs read off the target's
|
||||
req_to_token (VIRTUAL). Under fusion the draft pool takes the target's
|
||||
physical ids, so an untranslated write lands at the wrong row: the
|
||||
draft attends over the wrong KV and accept length collapses to 1.0
|
||||
with no crash. Each writer must hand the pool translated ids and leave
|
||||
the caller's locs virtual (the compact req_to_token rebuild re-reads
|
||||
them)."""
|
||||
"""DFLASH and DSPARK project the target's hidden states straight into
|
||||
the draft pool. Under fusion that pool takes the target's physical ids:
|
||||
both writers take them from the iteration's plan in the draft pool's
|
||||
space, never the virtual ids (a wrong row, with no crash)."""
|
||||
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
|
||||
from sglang.srt.speculative.dspark_components.dspark_kv_inject import (
|
||||
TargetHiddenKvInjector,
|
||||
@@ -317,11 +179,15 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
|
||||
is_translating = True
|
||||
|
||||
def translate_full_attn_ids(self, ids, *, out=None):
|
||||
return ids + 100
|
||||
|
||||
virtual = torch.tensor([4, 5, 6, 9], dtype=torch.int64)
|
||||
physical = virtual + 100
|
||||
draft_translator = _Shift()
|
||||
# The iteration's plan: physical ids for the fused draft's translator.
|
||||
plan = SimpleNamespace(
|
||||
write_ids=lambda reader, cols=None: (
|
||||
physical.clone() if reader is draft_translator else virtual.clone()
|
||||
)
|
||||
)
|
||||
hidden = torch.randn(4, 8)
|
||||
positions = torch.arange(4)
|
||||
commit_lens = torch.tensor([2, 1], dtype=torch.int32)
|
||||
@@ -348,7 +214,7 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
worker = SimpleNamespace(
|
||||
model_runner=SimpleNamespace(device=torch.device(_DEV)),
|
||||
draft_model_runner=SimpleNamespace(
|
||||
kv_index_translator=_Shift(), token_to_kv_pool=_DflashPool()
|
||||
kv_index_translator=draft_translator, token_to_kv_pool=_DflashPool()
|
||||
),
|
||||
draft_model=SimpleNamespace(
|
||||
project_target_hidden=lambda h: h,
|
||||
@@ -365,30 +231,29 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
)
|
||||
|
||||
# Per-token writes (prefill).
|
||||
cache_loc = virtual.clone()
|
||||
cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan)
|
||||
DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc(
|
||||
worker, target_hidden=hidden, cache_loc=cache_loc, positions=positions
|
||||
)
|
||||
self.assertTrue(writes["loc"].physical)
|
||||
torch.testing.assert_close(writes["loc"].loc, physical, rtol=0, atol=0)
|
||||
torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0)
|
||||
|
||||
# Prefix-valid writes (post-verify), from the 2-D view of the same ids.
|
||||
cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2)
|
||||
cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan)
|
||||
DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc(
|
||||
worker,
|
||||
target_hidden=hidden,
|
||||
cache_loc=cache_loc,
|
||||
positions=positions,
|
||||
cache_loc_2d=cache_loc_2d,
|
||||
cache_loc_2d=cache_loc.view(2, 2),
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
writes["loc_2d"], physical.view(2, 2), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0)
|
||||
|
||||
# DSPARK's injector, on an MHA draft pool.
|
||||
# DSPARK's injector, on an MHA draft pool: the plan's ids for the
|
||||
# draft's translator, written as they are.
|
||||
injected = {}
|
||||
|
||||
def write_target_hidden_kv(**kwargs):
|
||||
@@ -397,50 +262,47 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
injector = TargetHiddenKvInjector(
|
||||
draft_model=SimpleNamespace(write_target_hidden_kv=write_target_hidden_kv),
|
||||
draft_model_runner=SimpleNamespace(
|
||||
kv_index_translator=_Shift(), token_to_kv_pool=SimpleNamespace()
|
||||
kv_index_translator=draft_translator, token_to_kv_pool=SimpleNamespace()
|
||||
),
|
||||
model_runner=SimpleNamespace(device=torch.device(_DEV)),
|
||||
device=torch.device(_DEV),
|
||||
verify_num_draft_tokens=2,
|
||||
block_pos_offsets=torch.arange(2),
|
||||
)
|
||||
cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2)
|
||||
cache_loc = injector.ids_for(plan)
|
||||
injector.inject_target_hidden(
|
||||
target_hidden=hidden,
|
||||
cache_loc=cache_loc,
|
||||
positions=positions,
|
||||
cache_loc_2d=cache_loc_2d,
|
||||
cache_loc_2d=cache_loc.view(2, 2),
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
torch.testing.assert_close(injected["cache_loc"], physical, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
injected["cache_loc_2d"], physical.view(2, 2), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0)
|
||||
torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0)
|
||||
|
||||
def test_full_flat_v2p_per_disposition(self):
|
||||
"""The flat-translate accessor must hand a kernel exactly what
|
||||
def test_full_v2p_table_per_disposition(self):
|
||||
"""The v2p accessor must hand a DCP table builder exactly what
|
||||
`translate_kv_loc` would use: the shared v2p table on the target and
|
||||
on the fused draft, and None on a pass-through runner (a translated
|
||||
pass-through would address a private buffer with the target's ids)."""
|
||||
_, allocator, kvcache, draft_pool = _build()
|
||||
|
||||
target = _source(allocator, kvcache)
|
||||
self.assertIs(target.full_flat_v2p(), allocator.full_v2p_page_table)
|
||||
self.assertIs(target.full_v2p_table, allocator.full_v2p_page_table)
|
||||
|
||||
draft = _source(allocator, draft_pool)
|
||||
self.assertIs(draft.full_flat_v2p(), allocator.full_v2p_page_table)
|
||||
self.assertIs(draft.full_v2p_table, allocator.full_v2p_page_table)
|
||||
|
||||
passthrough = _source(allocator, _FakeKVCache(64))
|
||||
self.assertIsNone(passthrough.full_flat_v2p())
|
||||
self.assertIsNone(passthrough.full_v2p_table)
|
||||
|
||||
def test_multi_step_containers_pass_the_v2p_table_to_the_kernel(self):
|
||||
"""Every `generate_draft_decode_kv_indices` launch must thread the
|
||||
runner's v2p table and a matching TRANSLATE flag. A launch
|
||||
without them emits raw req_to_token values, which under the unified
|
||||
pool are VIRTUAL ids the fused draft pool cannot address — the exact
|
||||
silent-garbage bug this series fixed."""
|
||||
def test_multi_step_containers_read_the_plans_table(self):
|
||||
"""Every `generate_draft_decode_kv_indices` launch must gather from
|
||||
`read_source` with its entry granularity. A launch over raw
|
||||
req_to_token emits VIRTUAL ids, which the fused draft pool cannot
|
||||
address -- silent garbage drafts."""
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
@@ -454,112 +316,20 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
|
||||
if launches:
|
||||
launching[path.name] = (
|
||||
launches,
|
||||
len(re.findall(r"TRANSLATE=v2p is not None", text)),
|
||||
"full_flat_v2p" in text,
|
||||
len(re.findall(r"ENTRY_PAGE_SIZE=src\.entry_page_size", text)),
|
||||
"kv_index_translator.read_source(" in text,
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
len(launching), 4, f"launch sites disappeared: {sorted(launching)}"
|
||||
)
|
||||
for name, (launches, translated, has_accessor) in launching.items():
|
||||
for name, (launches, granular, has_source) in launching.items():
|
||||
self.assertEqual(
|
||||
launches,
|
||||
translated,
|
||||
f"{name}: {launches} kernel launch(es) but only {translated} "
|
||||
"carry TRANSLATE=v2p is not None",
|
||||
granular,
|
||||
f"{name}: {launches} kernel launch(es) but only {granular} "
|
||||
"pass the read table's entry granularity",
|
||||
)
|
||||
self.assertTrue(
|
||||
has_accessor,
|
||||
f"{name} launches the kernel without full_flat_v2p",
|
||||
)
|
||||
|
||||
|
||||
class TestDispositionBranchesAgree(unittest.TestCase):
|
||||
"""Every `__init__` disposition must assign the SAME attribute set.
|
||||
|
||||
BUG REGRESSION. `KVIndexTranslator.__init__` picks a disposition --
|
||||
translating (unified target or fused draft) or passthrough -- and later
|
||||
methods read attributes off `self` unconditionally. When upstream adds an
|
||||
attribute it naturally adds it to the branches IT knows about; a branch
|
||||
added here is silently left short, and nothing fails until the missing
|
||||
attribute is read at RUNTIME. That is exactly how the fused-draft branch
|
||||
lost `_translate_write_full` and `defer_read_translate` across a rebase:
|
||||
py_compile passes (the attribute is only ever read, never declared), the
|
||||
undefined-NAME check passes (it is an attribute, not a bare name), and the
|
||||
target and passthrough paths both work -- only a fused-draft forward raises
|
||||
`AttributeError: 'KVIndexTranslator' object has no attribute
|
||||
'_translate_write_full'`.
|
||||
|
||||
Comparing the branches against EACH OTHER needs no list to maintain: a new
|
||||
attribute is covered the moment any one branch sets it.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
|
||||
# parallel width that only exists once a config is published.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
def _init_branch_assignments(self):
|
||||
"""{branch index: {attr names it assigns}} for __init__'s if/elif/else."""
|
||||
src = pathlib.Path(kv_index_translator.__file__).read_text()
|
||||
tree = ast.parse(src)
|
||||
cls = next(
|
||||
n
|
||||
for n in ast.walk(tree)
|
||||
if isinstance(n, ast.ClassDef) and n.name == "KVIndexTranslator"
|
||||
)
|
||||
init = next(
|
||||
n
|
||||
for n in cls.body
|
||||
if isinstance(n, ast.FunctionDef) and n.name == "__init__"
|
||||
)
|
||||
|
||||
# The disposition chain is the `if` whose body assigns the id-space
|
||||
# fields; find it by the attribute every disposition must set.
|
||||
def assigns(stmts):
|
||||
out = set()
|
||||
for st in stmts:
|
||||
for node in ast.walk(st):
|
||||
if isinstance(node, ast.Assign):
|
||||
for t in node.targets:
|
||||
if (
|
||||
isinstance(t, ast.Attribute)
|
||||
and isinstance(t.value, ast.Name)
|
||||
and t.value.id == "self"
|
||||
):
|
||||
out.add(t.attr)
|
||||
return out
|
||||
|
||||
for node in init.body:
|
||||
if not isinstance(node, ast.If):
|
||||
continue
|
||||
branches, cur = [], node
|
||||
while True:
|
||||
branches.append(assigns(cur.body))
|
||||
if len(cur.orelse) == 1 and isinstance(cur.orelse[0], ast.If):
|
||||
cur = cur.orelse[0]
|
||||
continue
|
||||
if cur.orelse:
|
||||
branches.append(assigns(cur.orelse))
|
||||
break
|
||||
if len(branches) >= 2 and all("_full_v2p_table" in b for b in branches):
|
||||
return branches
|
||||
self.fail("could not locate the disposition if/elif/else in __init__")
|
||||
|
||||
def test_every_disposition_assigns_the_same_attributes(self):
|
||||
branches = self._init_branch_assignments()
|
||||
union = set().union(*branches)
|
||||
missing = {i: sorted(union - b) for i, b in enumerate(branches) if union - b}
|
||||
self.assertEqual(
|
||||
missing,
|
||||
{},
|
||||
"a KVIndexTranslator.__init__ disposition does not assign every "
|
||||
"attribute its siblings do; the branch raises AttributeError only "
|
||||
"when that path is taken at runtime: " + repr(missing),
|
||||
)
|
||||
self.assertTrue(has_source, f"{name} launches without read_source")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,745 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`KVLocPlan`: one iteration's KV ids, translated once.
|
||||
|
||||
- the target and a fused draft get the same physical write ids, a
|
||||
pass-through reader (a private draft pool, a static pool) the virtual
|
||||
window, which is never mutated; `bind` and a split batch's `cols_slice`
|
||||
pick the same tokens in both spaces;
|
||||
- a forward whose `seq_lens` do not yet count the window it writes (a
|
||||
verify, a speculative draft decode, a draft extend) reads that window past
|
||||
them;
|
||||
- one iteration's forwards translate the window once per sub-pool and build
|
||||
a sub-pool's read table at most once, only when a table reader asks; the
|
||||
table covers `seq_lens + read_extent`, padded lanes reading the sink;
|
||||
- only readers of the plan's own rows read its table; a pass-through reader
|
||||
gathers `req_to_token`, and a draft with its own `req_to_token` plans its
|
||||
own reads over the shared write ids (`reads_from`);
|
||||
- a runner's own write buffer and a replayed graph's padded buffer are used
|
||||
as they are, the sliding-window ids naming the sink past the plan's.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_kv_loc_plan.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache import kv_index_translator
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, TokenSpan
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
DenseDraftRegion,
|
||||
FusedDraftPlacement,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
_DEV = "cpu"
|
||||
_PS = 2
|
||||
_FULL = IdSpaceKind.FULL
|
||||
_SWA = IdSpaceKind.SLIDING_WINDOW
|
||||
|
||||
|
||||
def _reference(req_to_token, rows, lens, v2p, width):
|
||||
"""Independent derivation of a read table: each row's pages over `lens`,
|
||||
through `v2p`, the sink past them."""
|
||||
out = torch.zeros((rows.numel(), width), dtype=torch.int32)
|
||||
for b, row in enumerate(rows.tolist()):
|
||||
for c in range(min(-(-int(lens[b]) // _PS), width)):
|
||||
out[b, c] = max(int(v2p[int(req_to_token[row, c * _PS]) // _PS]), 0)
|
||||
return out
|
||||
|
||||
|
||||
class _HostStreamGather:
|
||||
"""`create_flashinfer_kv_indices_triton` on the host, for these CPU tests:
|
||||
lane ``b``'s ids from row ``row_ids[b]`` of the source, packed at
|
||||
``indptr[b]``."""
|
||||
|
||||
def __getitem__(self, grid):
|
||||
return self
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
ids,
|
||||
row_ids,
|
||||
seq_lens,
|
||||
indptr,
|
||||
kv_start_idx,
|
||||
out,
|
||||
row_stride,
|
||||
ENTRY_PAGE_SIZE=1,
|
||||
token_mapping=None,
|
||||
):
|
||||
assert row_stride == ids.stride(0)
|
||||
for b in range(int(seq_lens.numel())):
|
||||
start = 0 if kv_start_idx is None else int(kv_start_idx[b])
|
||||
pos = torch.arange(start, start + int(seq_lens[b]))
|
||||
entry = ids[int(row_ids[b]), pos // ENTRY_PAGE_SIZE].to(torch.int64)
|
||||
tok = entry * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
|
||||
if token_mapping is not None:
|
||||
tok = token_mapping[tok]
|
||||
lo = int(indptr[b])
|
||||
out[lo : lo + int(seq_lens[b])] = tok.to(out.dtype)
|
||||
|
||||
|
||||
class _FakeKVCache:
|
||||
def __init__(self, max_slots):
|
||||
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
|
||||
self.allocator = None
|
||||
|
||||
def attach_allocator(self, allocator):
|
||||
self.allocator = allocator
|
||||
|
||||
|
||||
class _FakeUnifiedSWAKVPool:
|
||||
def __init__(self, shared_pool):
|
||||
self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full"))
|
||||
self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa"))
|
||||
self.full_to_swa_index_mapping = None
|
||||
|
||||
def attach_allocators(self, *, full_allocator, swa_allocator):
|
||||
self._full_allocator = full_allocator
|
||||
self._swa_allocator = swa_allocator
|
||||
|
||||
|
||||
class TestKVLocPlan(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
|
||||
# parallel width that only exists once a config is published.
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||
|
||||
full_spec = MHASubPoolSpec(
|
||||
name="full",
|
||||
layer_num=2,
|
||||
head_num=2,
|
||||
head_dim=4,
|
||||
store_dtype=torch.bfloat16,
|
||||
grow_direction="down",
|
||||
draft_region=DenseDraftRegion(
|
||||
lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16
|
||||
),
|
||||
)
|
||||
swa_spec = MHASubPoolSpec(
|
||||
name="swa",
|
||||
layer_num=1,
|
||||
head_num=2,
|
||||
head_dim=4,
|
||||
store_dtype=torch.bfloat16,
|
||||
grow_direction="up",
|
||||
)
|
||||
n_full, n_swa = 32, 16
|
||||
pool = UnifiedKVPool(
|
||||
total_bytes=n_full * full_spec.entry_bytes()
|
||||
+ n_swa * swa_spec.entry_bytes(),
|
||||
sub_pool_specs=[full_spec, swa_spec],
|
||||
device=_DEV,
|
||||
enable_memory_saver=False,
|
||||
page_size=_PS,
|
||||
fused_draft=FusedDraftPlacement(
|
||||
region=full_spec.draft_region, runner_lane_counts=(1,)
|
||||
),
|
||||
)
|
||||
kvcache = _FakeUnifiedSWAKVPool(pool)
|
||||
self.allocator = UnifiedSWATokenToKVPoolAllocator(
|
||||
unified_buffer=pool,
|
||||
kvcache=kvcache,
|
||||
device=_DEV,
|
||||
full_max_total_num_tokens=n_full,
|
||||
swa_max_total_num_tokens=n_swa,
|
||||
page_size=_PS,
|
||||
need_sort=False,
|
||||
forward_stream=None,
|
||||
)
|
||||
draft_pool = UnifiedDraftKVPool(
|
||||
unified_buffer=pool,
|
||||
host_sub_pool_name="full",
|
||||
host_allocator=self.allocator,
|
||||
layer_lanes={0: 0},
|
||||
page_size=_PS,
|
||||
)
|
||||
# Every runner of one server shares the target's req_to_token.
|
||||
self.req_to_token = torch.zeros((4, 16), dtype=torch.int32)
|
||||
self.draft_pool = draft_pool
|
||||
self.target = self._translator(kvcache)
|
||||
self.fused_draft = self._translator(draft_pool)
|
||||
self.private_draft = self._translator(_FakeKVCache(64))
|
||||
|
||||
# Two requests, two pages each; the second page of each is this
|
||||
# iteration's write window (one token per request).
|
||||
self.rpi = torch.tensor([1, 2], dtype=torch.int64)
|
||||
for row in (1, 2):
|
||||
ids = self.allocator.alloc(2 * _PS)
|
||||
self.assertIsNotNone(ids)
|
||||
self.req_to_token[row, : 2 * _PS] = ids.to(torch.int32)
|
||||
self.seq_lens = torch.tensor([3, 3], dtype=torch.int64)
|
||||
self.window = self.req_to_token[self.rpi, 2:4].to(torch.int64).reshape(-1)
|
||||
|
||||
def _translator(self, pool):
|
||||
return KVIndexTranslator(
|
||||
req_to_token=self.req_to_token,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
token_to_kv_pool=pool,
|
||||
page_size=_PS,
|
||||
device=_DEV,
|
||||
)
|
||||
|
||||
def _plan(self, source=None, read_extent=1):
|
||||
return (source or self.target).plan(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=self.seq_lens.clone(),
|
||||
write_virtual=self.window,
|
||||
read_extent=read_extent,
|
||||
)
|
||||
|
||||
def test_readers_get_ids_in_their_own_space(self):
|
||||
window = self.window.clone()
|
||||
plan = self._plan()
|
||||
physical = self.allocator.translate_write_loc(window)
|
||||
self.assertTrue(torch.equal(plan.write_ids(self.target), physical))
|
||||
# A fused draft shares the target's pages: the same tensor, untranslated.
|
||||
self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical)
|
||||
# A private draft pool indexes virtual ids.
|
||||
self.assertIs(plan.write_ids(self.private_draft), self.window)
|
||||
self.assertTrue(torch.equal(self.window, window)) # never mutated
|
||||
|
||||
def test_cols_slice_names_a_split_batch_s_tokens(self):
|
||||
"""Two-batch overlap splits a forward's tokens; each half writes its
|
||||
own tokens of the window, whatever the forward's own columns were."""
|
||||
plan = self._plan()
|
||||
bs = int(self.rpi.numel())
|
||||
everything = plan.cols_slice(None, slice(1, None))
|
||||
self.assertEqual(everything, TokenSpan(1, self.window.numel()))
|
||||
self.assertEqual(plan.cols_slice(everything, slice(1, 3)), TokenSpan(2, 4))
|
||||
# A static plan's halves, and a translating one's, both read views.
|
||||
static = self._plan(self.private_draft)
|
||||
for p, reader in ((plan, self.target), (static, self.private_draft)):
|
||||
half = p.cols_slice(None, slice(0, 2))
|
||||
self.assertIsInstance(half, TokenSpan)
|
||||
whole = p.write_ids(reader)
|
||||
ids = p.write_ids(reader, cols=half)
|
||||
self.assertTrue(torch.equal(ids, whole[:2]))
|
||||
self.assertEqual(
|
||||
ids.untyped_storage().data_ptr(), whole.untyped_storage().data_ptr()
|
||||
)
|
||||
first = plan.cols_slice(slice(0, 1), slice(None))
|
||||
width = self.window.numel() // bs
|
||||
self.assertTrue(torch.equal(first, torch.arange(bs) * width))
|
||||
ragged = torch.tensor([3, -1, 0])
|
||||
self.assertTrue(torch.equal(plan.cols_slice(ragged, slice(1, 3)), ragged[1:]))
|
||||
for cols, tokens in (
|
||||
(None, slice(1, None)),
|
||||
(everything, slice(1, 3)),
|
||||
(slice(0, 1), slice(None)),
|
||||
):
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
plan.write_ids(self.target, cols=plan.cols_slice(cols, tokens)),
|
||||
plan.write_ids(self.target, cols=cols)[tokens],
|
||||
)
|
||||
)
|
||||
|
||||
def test_one_iteration_translates_once(self):
|
||||
writes, builds = [], []
|
||||
real_write = self.allocator.translate_write_loc
|
||||
real_build = kv_index_translator.build_kv_read_table
|
||||
|
||||
swa_writes = []
|
||||
real_swa_write = self.allocator.translate_loc_from_full_to_swa
|
||||
|
||||
def counting_write(ids, *a, **kw):
|
||||
writes.append(ids)
|
||||
return real_write(ids, *a, **kw)
|
||||
|
||||
def counting_swa_write(ids):
|
||||
swa_writes.append(ids)
|
||||
return real_swa_write(ids)
|
||||
|
||||
def counting_build(**kwargs):
|
||||
builds.append(kwargs["v2p"])
|
||||
return real_build(**kwargs)
|
||||
|
||||
with patch.object(kv_index_translator, "build_kv_read_table", counting_build):
|
||||
# The spaces hold the allocator's translates from when the
|
||||
# translator was built; count through them. The fused draft's full
|
||||
# space is the target's, so it counts there too.
|
||||
for reader, kind, write in (
|
||||
(self.target, _FULL, counting_write),
|
||||
(self.fused_draft, _FULL, counting_write),
|
||||
(self.target, _SWA, counting_swa_write),
|
||||
):
|
||||
reader._spaces[kind] = msgspec.structs.replace(
|
||||
reader.space(kind), write=write
|
||||
)
|
||||
plan = self._plan(read_extent=2)
|
||||
draft, verify, extend = (SimpleNamespace() for _ in range(3))
|
||||
plan.bind(draft, self.fused_draft, cols=slice(0, 1))
|
||||
plan.bind(verify, self.target)
|
||||
plan.bind(extend, self.fused_draft)
|
||||
bs = int(self.rpi.numel())
|
||||
for reader in (self.target, self.fused_draft):
|
||||
# What a packed stream gathers from, and a captured table.
|
||||
reader.read_source(plan, req_pool_indices=self.rpi, bs=bs + 1)
|
||||
reader.read_table(plan)
|
||||
reader.copy_page_table(
|
||||
plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32)
|
||||
)
|
||||
# The target's sliding-window layers, written and read twice.
|
||||
for _ in range(2):
|
||||
self.target.write_ids(verify, _SWA)
|
||||
self.target.read_table(plan, kind=_SWA)
|
||||
self.target.copy_page_table(
|
||||
plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32), kind=_SWA
|
||||
)
|
||||
self.assertEqual(len(writes), 1)
|
||||
self.assertEqual(len(swa_writes), 1)
|
||||
# The full and the sliding-window id space, one build each.
|
||||
self.assertEqual(len(builds), 2)
|
||||
self.assertTrue(torch.equal(verify.out_cache_loc, plan.write_physical))
|
||||
self.assertIs(extend.out_cache_loc, verify.out_cache_loc)
|
||||
|
||||
def test_a_verify_reads_the_window_it_writes(self):
|
||||
def own_plan(mode, spec_info):
|
||||
return self.target.own_plan(
|
||||
SimpleNamespace(
|
||||
forward_mode=mode,
|
||||
spec_info=spec_info,
|
||||
batch_size=int(self.rpi.numel()),
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=self.seq_lens.clone(),
|
||||
out_cache_loc=self.window,
|
||||
)
|
||||
)
|
||||
|
||||
width = self.window.numel() // self.rpi.numel()
|
||||
# A verify's rows read up to `draft_token_num` past their lengths, more
|
||||
# than the window when the verify is ragged.
|
||||
spec = SimpleNamespace(draft_token_num=width + 1)
|
||||
self.assertEqual(
|
||||
own_plan(ForwardMode.TARGET_VERIFY, spec).read_extent, width + 1
|
||||
)
|
||||
self.assertEqual(own_plan(ForwardMode.DECODE, spec).read_extent, width)
|
||||
# A draft extend's lengths take its window after its batch is built.
|
||||
self.assertEqual(own_plan(ForwardMode.DRAFT_EXTEND_V2, spec).read_extent, width)
|
||||
# Their lengths already count what an extend or a plain decode writes.
|
||||
self.assertEqual(own_plan(ForwardMode.EXTEND, spec).read_extent, 0)
|
||||
self.assertEqual(own_plan(ForwardMode.DECODE, None).read_extent, 0)
|
||||
|
||||
def test_a_stream_only_iteration_builds_no_table(self):
|
||||
"""A plan builds a table only when a table reader asks for it: stream
|
||||
readers -- one runner, or a target and a fused draft over the same
|
||||
rows -- each translate in their own gather. Once the table is built,
|
||||
a stream packs the same ids from it."""
|
||||
calls = {"packed": 0, "table": 0}
|
||||
|
||||
def counting(name, real):
|
||||
def wrapped(**kwargs):
|
||||
calls[name] += 1
|
||||
return real(**kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
# A runner of rows no other translator here reads.
|
||||
self.req_to_token = self.req_to_token.clone()
|
||||
solo = self._translator(self.allocator.get_kvcache())
|
||||
bs = int(self.rpi.numel())
|
||||
lens = torch.tensor([3, 2], dtype=torch.int64)
|
||||
indptr = torch.tensor([0, 3, 5], dtype=torch.int32)
|
||||
|
||||
def pack(translator, plan, out, kind=_FULL):
|
||||
return translator.pack_read_stream(
|
||||
plan,
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=lens,
|
||||
indptr=indptr,
|
||||
out=out,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
kv_index_translator,
|
||||
"build_kv_read_table_packed",
|
||||
counting("packed", kv_index_translator.build_kv_read_table_packed),
|
||||
),
|
||||
patch.object(
|
||||
kv_index_translator,
|
||||
"build_kv_read_table",
|
||||
counting("table", kv_index_translator.build_kv_read_table),
|
||||
),
|
||||
patch.object(
|
||||
kv_index_translator,
|
||||
"create_flashinfer_kv_indices_triton",
|
||||
_HostStreamGather(),
|
||||
),
|
||||
):
|
||||
plan = self._plan(solo, read_extent=1)
|
||||
out = torch.full((6,), -7, dtype=torch.int32)
|
||||
self.assertTrue(pack(solo, plan, out))
|
||||
self.assertEqual(calls, {"packed": 1, "table": 0})
|
||||
self.assertFalse(plan.has_read_table())
|
||||
for b in range(bs):
|
||||
row = self.req_to_token[int(self.rpi[b]), : int(lens[b])]
|
||||
want = self.allocator.translate_write_loc(row.to(torch.int64))
|
||||
got = out[int(indptr[b]) : int(indptr[b + 1])].to(torch.int64)
|
||||
self.assertTrue(torch.equal(got, want))
|
||||
self.assertEqual(int(out[5]), -7) # past the stream: untouched
|
||||
|
||||
# A caller holding a padded graph buffer of request rows: the
|
||||
# lengths size the batch, the extra lanes are left out.
|
||||
padded = torch.cat([self.rpi, torch.zeros(5, dtype=torch.int64)])
|
||||
again = torch.full((6,), -7, dtype=torch.int32)
|
||||
solo.pack_read_stream(
|
||||
self._plan(solo, read_extent=1),
|
||||
req_pool_indices=padded,
|
||||
seq_lens=lens,
|
||||
indptr=indptr,
|
||||
out=again,
|
||||
)
|
||||
self.assertTrue(torch.equal(again, out))
|
||||
self.assertEqual(calls, {"packed": 2, "table": 0})
|
||||
|
||||
# Once a table reader has the table built, a stream packs from it:
|
||||
# nothing gathered again, the same ids.
|
||||
plan.read_table()
|
||||
after = torch.full((6,), -7, dtype=torch.int32)
|
||||
pack(solo, plan, after)
|
||||
self.assertEqual(calls, {"packed": 2, "table": 1})
|
||||
self.assertTrue(torch.equal(after, out))
|
||||
# Nothing asked for the sliding-window table: its stream comes
|
||||
# alone.
|
||||
pack(solo, plan, torch.empty(6, dtype=torch.int32), kind=_SWA)
|
||||
self.assertEqual(calls, {"packed": 3, "table": 1})
|
||||
self.assertFalse(plan.has_read_table(_SWA))
|
||||
|
||||
# Another stream reader of the same rows (a fused draft): still no
|
||||
# table -- each translates its own gather.
|
||||
peer = self._translator(self.draft_pool)
|
||||
for seq_lens_cpu in (self.seq_lens.clone(), None):
|
||||
both = solo.plan(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=self.window,
|
||||
read_extent=1,
|
||||
)
|
||||
pack(solo, both, torch.empty(6, dtype=torch.int32))
|
||||
pack(peer, both, torch.empty(6, dtype=torch.int32))
|
||||
# The draft's row gather gets the virtual rows and the page
|
||||
# table, translated in its own kernel.
|
||||
src = peer.read_source(both, req_pool_indices=self.rpi, bs=bs)
|
||||
self.assertTrue(src.is_translated)
|
||||
self.assertIs(src.ids, self.req_to_token)
|
||||
self.assertIs(src.v2p, solo.space(_FULL).read_v2p)
|
||||
self.assertFalse(both.has_read_table())
|
||||
self.assertEqual(calls, {"packed": 7, "table": 1})
|
||||
|
||||
def test_a_captured_first_reader_holds_the_table(self):
|
||||
"""When the plan's first reader is a captured table, the table is built
|
||||
in that buffer -- no table of the plan's own, no copy -- and serves the
|
||||
plan's other readers; the same buffer again is left alone, and another
|
||||
captured reader copies it rather than building again; an eager reader
|
||||
of the batch takes the batch's rows of it."""
|
||||
builds = []
|
||||
real = kv_index_translator.build_kv_read_table
|
||||
|
||||
def counting(**kwargs):
|
||||
builds.append(kwargs["out"])
|
||||
return real(**kwargs)
|
||||
|
||||
bs = int(self.rpi.numel())
|
||||
captured = torch.full((bs + 1, 8), 7, dtype=torch.int32)
|
||||
with patch.object(kv_index_translator, "build_kv_read_table", counting):
|
||||
plan = self._plan(read_extent=1)
|
||||
self.target.copy_page_table(plan, out=captured)
|
||||
self.assertEqual(len(builds), 1)
|
||||
self.assertEqual(builds[0].data_ptr(), captured.data_ptr())
|
||||
table = plan.read_table(rows=bs + 1)
|
||||
self.assertEqual(table.ids.data_ptr(), captured.data_ptr())
|
||||
self.assertEqual(table.row_stride, captured.stride(0))
|
||||
# An eager reader of the batch (fa3, trtllm_mha) gets the batch's
|
||||
# rows of that table: its kernels size the batch by them.
|
||||
eager = self.target.read_table(plan, rows=bs)
|
||||
self.assertEqual(eager.ids.shape[0], bs)
|
||||
self.assertEqual(eager.row_ids.numel(), bs)
|
||||
self.assertEqual(eager.ids.data_ptr(), captured.data_ptr())
|
||||
width = table.ids.shape[1]
|
||||
reference = _reference(
|
||||
self.req_to_token,
|
||||
self.rpi,
|
||||
self.seq_lens + 1,
|
||||
self.allocator.full_v2p_page_table,
|
||||
width,
|
||||
)
|
||||
self.assertTrue(torch.equal(captured[:bs, :width], reference))
|
||||
self.assertEqual(int(captured[bs, :width].abs().sum()), 0) # padded lane
|
||||
self.assertTrue(bool((captured[:, width:] == 7).all())) # past the width
|
||||
self.target.copy_page_table(plan, out=captured)
|
||||
other = torch.full_like(captured, 5)
|
||||
self.target.copy_page_table(plan, out=other)
|
||||
self.assertEqual(len(builds), 1)
|
||||
self.assertTrue(torch.equal(other[:, :width], captured[:, :width]))
|
||||
|
||||
def test_a_captured_table_bounds_a_build_without_host_lengths(self):
|
||||
"""Without host lengths, a captured first reader narrower than a row
|
||||
has the table built in its own buffer, to its width."""
|
||||
builds = []
|
||||
real = kv_index_translator.build_kv_read_table
|
||||
|
||||
def counting(**kwargs):
|
||||
builds.append(kwargs["out"])
|
||||
return real(**kwargs)
|
||||
|
||||
bs = int(self.rpi.numel())
|
||||
row_pages = -(-self.req_to_token.shape[1] // _PS)
|
||||
captured = torch.full((bs + 1, row_pages // 2), 7, dtype=torch.int32)
|
||||
with patch.object(kv_index_translator, "build_kv_read_table", counting):
|
||||
plan = self.target.plan(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=None,
|
||||
write_virtual=self.window,
|
||||
read_extent=1,
|
||||
)
|
||||
self.target.copy_page_table(plan, out=captured)
|
||||
self.assertEqual(len(builds), 1)
|
||||
self.assertEqual(builds[0].data_ptr(), captured.data_ptr())
|
||||
self.assertEqual(builds[0].shape[1], captured.shape[1])
|
||||
reference = _reference(
|
||||
self.req_to_token,
|
||||
self.rpi,
|
||||
self.seq_lens + 1,
|
||||
self.allocator.full_v2p_page_table,
|
||||
captured.shape[1],
|
||||
)
|
||||
self.assertTrue(torch.equal(captured[:bs], reference))
|
||||
self.assertEqual(int(captured[bs].abs().sum()), 0) # padded lane
|
||||
|
||||
def test_read_table_is_built_once_and_padded_by_copy(self):
|
||||
builds = []
|
||||
real = kv_index_translator.build_kv_read_table
|
||||
|
||||
def counting(**kwargs):
|
||||
builds.append(kwargs["v2p"])
|
||||
return real(**kwargs)
|
||||
|
||||
with patch.object(kv_index_translator, "build_kv_read_table", counting):
|
||||
plan = self._plan(read_extent=1)
|
||||
table = plan.read_table()
|
||||
self.assertIs(plan.read_table(), table)
|
||||
# Only the space that was read.
|
||||
self.assertEqual(len(builds), 1)
|
||||
swa = plan.read_table(kind=_SWA)
|
||||
self.assertEqual(len(builds), 2)
|
||||
padded = plan.read_table(rows=4)
|
||||
padded_swa = plan.read_table(kind=_SWA, rows=4)
|
||||
self.assertEqual(len(builds), 2) # copied, not built again
|
||||
self.assertTrue(torch.equal(padded.ids[:2], table.ids))
|
||||
self.assertEqual(int(padded.ids[2:].abs().sum()), 0) # the sink
|
||||
self.assertTrue(torch.equal(padded_swa.ids[:2], swa.ids))
|
||||
self.assertEqual(int(padded_swa.ids[2:].abs().sum()), 0)
|
||||
# The rows cover `seq_lens + read_extent`.
|
||||
reference = _reference(
|
||||
self.req_to_token,
|
||||
self.rpi,
|
||||
self.seq_lens + 1,
|
||||
self.allocator.full_v2p_page_table,
|
||||
table.ids.shape[1],
|
||||
)
|
||||
self.assertTrue(torch.equal(table.ids, reference))
|
||||
|
||||
def test_only_readers_of_its_rows_read_its_table(self):
|
||||
plan = self._plan()
|
||||
self.assertTrue(plan.is_read_by(self.target))
|
||||
self.assertTrue(plan.is_read_by(self.fused_draft))
|
||||
self.assertFalse(plan.is_read_by(self.private_draft))
|
||||
compact = KVIndexTranslator(
|
||||
req_to_token=self.req_to_token.clone(),
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
token_to_kv_pool=self.draft_pool,
|
||||
page_size=_PS,
|
||||
device=_DEV,
|
||||
)
|
||||
self.assertFalse(plan.is_read_by(compact))
|
||||
# Before a table reader asks, the fused draft's row gather translates
|
||||
# in its own kernel; after, it reads the plan's table.
|
||||
src = self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2)
|
||||
self.assertIs(src.v2p, self.target.space(_FULL).read_v2p)
|
||||
self.assertFalse(plan.has_read_table())
|
||||
table = plan.read_table(rows=2)
|
||||
self.assertIs(
|
||||
self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2), table
|
||||
)
|
||||
passthrough = self.private_draft.read_source(
|
||||
plan, req_pool_indices=self.rpi, bs=2
|
||||
)
|
||||
self.assertFalse(passthrough.is_translated)
|
||||
self.assertIs(passthrough.ids, self.req_to_token)
|
||||
|
||||
def test_a_compact_draft_reads_its_own_rows_over_the_shared_writes(self):
|
||||
plan = self._plan()
|
||||
compact_rows = self.req_to_token.clone()
|
||||
compact = KVIndexTranslator(
|
||||
req_to_token=compact_rows,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
token_to_kv_pool=self.draft_pool,
|
||||
page_size=_PS,
|
||||
device=_DEV,
|
||||
)
|
||||
lens = torch.tensor([2, 1], dtype=torch.int64)
|
||||
derived = plan.reads_from(
|
||||
compact, seq_lens=lens, seq_lens_cpu=lens.clone(), read_extent=1
|
||||
)
|
||||
# The writes are the plan's, not translated again.
|
||||
self.assertIs(derived.write_physical, plan.write_physical)
|
||||
self.assertIs(derived.write_ids(compact), plan.write_physical)
|
||||
self.assertTrue(derived.is_read_by(compact))
|
||||
compact_rows[self.rpi, :2] = compact_rows[self.rpi, 2:4]
|
||||
reference = _reference(
|
||||
compact_rows,
|
||||
self.rpi,
|
||||
lens + 1,
|
||||
self.allocator.full_v2p_page_table,
|
||||
derived.read_table().ids.shape[1],
|
||||
)
|
||||
self.assertTrue(torch.equal(derived.read_table().ids, reference))
|
||||
|
||||
def test_bind_sets_the_write_fields(self):
|
||||
"""`bind` gives a batch the columns it writes, the same ones in both
|
||||
spaces, with the virtual mirror only when its ids were translated."""
|
||||
plan = self._plan()
|
||||
bs = int(self.rpi.numel())
|
||||
first = slice(0, 1)
|
||||
fused = SimpleNamespace()
|
||||
plan.bind(fused, self.fused_draft, cols=first)
|
||||
self.assertIs(fused.kv_loc_plan, plan)
|
||||
self.assertEqual(fused.kv_loc_cols, first)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
fused.out_cache_loc,
|
||||
plan.write_physical.view(bs, -1)[:, first].reshape(-1),
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
fused.out_cache_loc_virtual,
|
||||
self.window.view(bs, -1)[:, first].reshape(-1),
|
||||
)
|
||||
)
|
||||
self.assertTrue(fused.out_cache_loc_is_physical)
|
||||
private = SimpleNamespace()
|
||||
plan.bind(private, self.private_draft)
|
||||
self.assertIs(private.out_cache_loc, self.window)
|
||||
self.assertIsNone(private.out_cache_loc_virtual)
|
||||
|
||||
def test_an_npu_dcp_target_writes_rank_local_slots(self):
|
||||
"""On NPU with DCP a target writes its rank's slots, while the
|
||||
scheduler, `req_to_token` and the replicated draft pool keep
|
||||
allocator-global ones: a plan's window stays global whichever runner
|
||||
built it (a target prefill handing its plan to the draft, a draft
|
||||
handing its window to the verify), the target binds it rank-local,
|
||||
and the draft binds it global."""
|
||||
parallel = SimpleNamespace(
|
||||
dcp_enabled=True, dcp_size=2, dcp_rank=1, attn_dcp_size=1
|
||||
)
|
||||
with (
|
||||
patch.object(kv_index_translator, "_is_npu", True),
|
||||
patch.object(kv_index_translator, "get_parallel", return_value=parallel),
|
||||
):
|
||||
target, draft = (
|
||||
KVIndexTranslator(
|
||||
req_to_token=self.req_to_token,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
token_to_kv_pool=_FakeKVCache(64),
|
||||
page_size=4,
|
||||
device=_DEV,
|
||||
is_draft_worker=is_draft_worker,
|
||||
)
|
||||
for is_draft_worker in (False, True)
|
||||
)
|
||||
global_ids = torch.arange(8, 16)
|
||||
# Rank 1 of 2 owns every other 4-slot page, compacted.
|
||||
local_ids = torch.tensor([-1, -1, -1, -1, 4, 5, 6, 7])
|
||||
for source in (target, draft):
|
||||
plan = source.plan(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=self.seq_lens.clone(),
|
||||
write_virtual=global_ids,
|
||||
)
|
||||
target_batch, draft_batch = SimpleNamespace(), SimpleNamespace()
|
||||
plan.bind(target_batch, target)
|
||||
plan.bind(draft_batch, draft)
|
||||
self.assertTrue(torch.equal(target_batch.out_cache_loc, local_ids))
|
||||
self.assertTrue(torch.equal(draft_batch.out_cache_loc, global_ids))
|
||||
self.assertIs(plan.write_virtual, global_ids)
|
||||
|
||||
def test_runner_slots_are_used_as_they_are(self):
|
||||
for translator, writes_swa in (
|
||||
(self.target, True),
|
||||
(self.private_draft, False),
|
||||
):
|
||||
slots = torch.zeros(2, dtype=torch.int64)
|
||||
batch = SimpleNamespace(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=self.seq_lens.clone(),
|
||||
out_cache_loc=slots,
|
||||
)
|
||||
translator.bind_runner_slots(batch)
|
||||
# Kept by address: a captured graph writes this buffer on replay.
|
||||
self.assertIs(batch.out_cache_loc, slots)
|
||||
self.assertIsNone(batch.out_cache_loc_virtual)
|
||||
self.assertTrue(batch.out_cache_loc_is_physical)
|
||||
swa = translator.write_ids(batch, _SWA)
|
||||
if writes_swa:
|
||||
# The sink in the sliding-window sub-pool too.
|
||||
self.assertTrue(torch.equal(swa, torch.zeros_like(slots)))
|
||||
else:
|
||||
self.assertIsNone(swa)
|
||||
|
||||
def test_a_replay_batch_writes_the_padded_slots(self):
|
||||
plan = self._plan()
|
||||
n = plan.write_physical.numel()
|
||||
slots = torch.zeros(n + 2, dtype=torch.int64)
|
||||
slots[:n].copy_(plan.write_physical)
|
||||
batch = SimpleNamespace()
|
||||
plan.bind_replay(batch, self.target, slots=slots)
|
||||
self.assertIs(batch.kv_loc_plan, plan)
|
||||
self.assertIs(batch.out_cache_loc, slots)
|
||||
self.assertIsNone(batch.out_cache_loc_virtual)
|
||||
swa = self.target.write_ids(batch, _SWA)
|
||||
self.assertEqual(swa.shape[0], n + 2)
|
||||
self.assertTrue(torch.equal(swa[:n], plan.write_ids(self.target, kind=_SWA)))
|
||||
self.assertEqual(int(swa[n:].abs().sum()), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
Under the token-major views a virtual id is in range and, by value, the same
|
||||
kind of integer as a physical one, so a unified pool cannot tell a skipped
|
||||
rebind from a translated loc by looking at it. `rebind_write_loc` marks the
|
||||
bind from a translated loc by looking at it. A plan's `bind` marks the
|
||||
batch's loc physical, producers carry the mark in `KVWriteLoc`, composites
|
||||
forward it, and the unified write doors refuse a loc without it.
|
||||
|
||||
@@ -31,6 +31,7 @@ from types import SimpleNamespace
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
KVWriteLoc,
|
||||
MHATokenToKVPool,
|
||||
@@ -66,7 +67,11 @@ def _plain_translator():
|
||||
def _translating_translator(v2p):
|
||||
src = _plain_translator()
|
||||
src.is_translating = True
|
||||
src._translate_write_full = lambda t, out=None: v2p[t.to(torch.int64)]
|
||||
src._spaces = {
|
||||
IdSpaceKind.FULL: IdSpace(
|
||||
key=(IdSpaceKind.FULL, "test"), write=lambda t: v2p[t.to(torch.int64)]
|
||||
)
|
||||
}
|
||||
return src
|
||||
|
||||
|
||||
@@ -111,7 +116,7 @@ class TestRebindMarksTheBatch(unittest.TestCase):
|
||||
def test_non_translating_pool_marks_physical_without_rebinding(self):
|
||||
loc = torch.tensor([3, 5], dtype=torch.int64)
|
||||
fb = _batch(loc)
|
||||
_plain_translator().rebind_write_loc(fb)
|
||||
_plain_translator().bind_own_plan(fb)
|
||||
self.assertIs(fb.out_cache_loc, loc) # physical by allocation: untouched
|
||||
self.assertTrue(fb.out_cache_loc_is_physical)
|
||||
|
||||
@@ -119,13 +124,13 @@ class TestRebindMarksTheBatch(unittest.TestCase):
|
||||
v2p = torch.tensor([7, 6, 5, 4], dtype=torch.int64)
|
||||
loc = torch.tensor([1, 2], dtype=torch.int64)
|
||||
fb = _batch(loc)
|
||||
_translating_translator(v2p).rebind_write_loc(fb)
|
||||
_translating_translator(v2p).bind_own_plan(fb)
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, v2p[loc]))
|
||||
self.assertTrue(fb.out_cache_loc_is_physical)
|
||||
|
||||
def test_no_loc_stays_unmarked(self):
|
||||
fb = _batch(None)
|
||||
_translating_translator(torch.arange(4)).rebind_write_loc(fb)
|
||||
_translating_translator(torch.arange(4)).bind_own_plan(fb)
|
||||
self.assertFalse(fb.out_cache_loc_is_physical)
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ from types import SimpleNamespace
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_view
|
||||
from sglang.srt.mem_cache.layout.token_major import ENTRY_ALIGN_BYTES, build_dense_views
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
@@ -550,10 +551,10 @@ class TestFactoryViews(unittest.TestCase):
|
||||
self.assertEqual(pool.k_buffer[0].dtype, torch.uint8)
|
||||
|
||||
def test_rebind_emits_physical_full_and_build_derives_swa(self):
|
||||
"""End-to-end over the real factory: rebind_write_loc rebinds
|
||||
out_cache_loc to FULL-side physical ids (phase 1), and the per-batch
|
||||
build derives the SWA write loc pointwise from those values (phase 2)
|
||||
-- both checked against the v2p tables over the VIRTUAL ids."""
|
||||
"""End-to-end over the real factory: a batch's own plan binds
|
||||
out_cache_loc to FULL-side physical ids and derives the SWA write ids
|
||||
from the virtual window -- both checked against the v2p tables over
|
||||
the VIRTUAL ids."""
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -584,11 +585,11 @@ class TestFactoryViews(unittest.TestCase):
|
||||
device="cpu",
|
||||
)
|
||||
self.assertTrue(source.is_translating)
|
||||
source.rebind_write_loc(fb)
|
||||
source.bind_own_plan(fb)
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, expected_full))
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
source.sliding_window_write_loc_for(fb.out_cache_loc), expected_swa
|
||||
source.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), expected_swa
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -21,16 +21,15 @@ page ids:
|
||||
|
||||
kernel_page(virtual_page) = v2p[virtual_page]
|
||||
|
||||
Since the read-path translator, ONE builder computes that formula for every
|
||||
family — `build_index_table` (the canonical) — and the backends only
|
||||
differ in how they consume it:
|
||||
- trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: rows filled straight
|
||||
into their padded block tables (`KVIndexTranslator.fill_read_table`, prefix-only so
|
||||
the backends' own -1 / stale tail sentinels survive);
|
||||
- the flashinfer updaters: token ids reconstructed from the canonical by
|
||||
ONE builder computes that formula for every family -- the iteration plan's
|
||||
read table (`KVLocPlan.read_table`) -- and the backends only differ in how
|
||||
they consume it:
|
||||
- trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows
|
||||
copied into their padded block tables (`KVIndexTranslator.copy_page_table`);
|
||||
- the flashinfer updaters: token ids reconstructed from the table by
|
||||
`create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`;
|
||||
- fa3's captured decode: `normal_decode_set_metadata` copies the canonical
|
||||
rows' live prefixes (src_is_read_table=True).
|
||||
- fa3's captured decode: the plan's rows copied into its captured page
|
||||
table.
|
||||
|
||||
Covered here:
|
||||
- the static `create_flashmla_kv_indices_triton` (no id-space knowledge left)
|
||||
@@ -216,6 +215,44 @@ class TestBlockTable(unittest.TestCase):
|
||||
torch.equal(gpu, cpu), f"ps={page_size}:\ngpu={gpu}\ncpu={cpu}"
|
||||
)
|
||||
|
||||
def test_zero_tail_fills_a_fresh_table(self):
|
||||
"""With ``zero_tail`` a fresh (garbage) table comes back whole in one
|
||||
launch: each row's live prefix, the sink past it up to ``max_pages``,
|
||||
and nothing past ``max_pages``. The CUDA kernel must agree with the
|
||||
CPU path."""
|
||||
from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table
|
||||
|
||||
for page_size in (1, 32):
|
||||
rt, rpi, sl, v2p = self._make_batch(page_size)
|
||||
bs = rpi.shape[0]
|
||||
max_pages = int((sl.max().item() + page_size - 1) // page_size) + 2
|
||||
|
||||
def fill(device):
|
||||
# One guard column past the table catches a spill off the row.
|
||||
out = torch.full(
|
||||
(bs, max_pages + 1), 7, dtype=torch.int32, device=device
|
||||
)
|
||||
build_kv_read_table(
|
||||
req_to_token=rt.to(device),
|
||||
req_pool_indices=rpi.to(device),
|
||||
seq_lens=sl.to(device=device, dtype=torch.int64),
|
||||
v2p=v2p.to(device),
|
||||
page_size=page_size,
|
||||
max_pages=max_pages,
|
||||
out=out,
|
||||
zero_tail=True,
|
||||
)
|
||||
return out.cpu()
|
||||
|
||||
gpu, cpu = fill(_DEV), fill("cpu")
|
||||
self.assertTrue(torch.equal(gpu, cpu), f"ps={page_size}")
|
||||
self.assertTrue(bool((gpu[:, max_pages] == 7).all()), "spilled")
|
||||
want = _reference(rt, rpi, sl, page_size, v2p=v2p).cpu().to(torch.int32)
|
||||
for b in range(bs):
|
||||
live = int((int(sl[b]) + page_size - 1) // page_size)
|
||||
self.assertTrue(torch.equal(gpu[b, :live], want[b, :live]))
|
||||
self.assertTrue(bool((gpu[b, live:max_pages] == 0).all()))
|
||||
|
||||
def test_static_kernel_matches_reference(self):
|
||||
"""The stripped (id-space-free) flashmla kernel is byte-identical to the
|
||||
plain token//ps reference -- guards the v2p-arg removal itself."""
|
||||
|
||||
@@ -303,10 +303,8 @@ class TestDefaultConfigurator(CustomTestCase):
|
||||
self.assertEqual(mock_calculate_mla_kv_cache_dim.call_count, 2)
|
||||
|
||||
def test_fused_draft_entry_overrides_the_layer_ratio(self):
|
||||
"""Unified mamba-MHA fusion prices the EXACT fused entry (host +
|
||||
draft + alignment pad): the per-draft-layer ratio under-reserves the pad,
|
||||
so a solve over it hands out more tokens than the fused pages hold.
|
||||
The control pins that path-only drafts keep the ratio arm."""
|
||||
"""A fused draft prices the exact fused entry (host + draft + pad);
|
||||
the per-draft-layer ratio under-reserves the pad."""
|
||||
mr = _make_model_runner(self, speculative_algorithm="EAGLE")
|
||||
mr.spec_algorithm.is_eagle.return_value = True
|
||||
mr.spec_algorithm.is_none.return_value = False
|
||||
@@ -320,12 +318,7 @@ class TestDefaultConfigurator(CustomTestCase):
|
||||
)
|
||||
|
||||
fused = create_memory_pool_configurator(mr)
|
||||
mr.fused_entry_bytes.side_effect = None
|
||||
mr.fused_entry_bytes.return_value = None
|
||||
unfused = create_memory_pool_configurator(mr)
|
||||
self.assertEqual(fused._cell_size, 77777)
|
||||
base = _full_per_token(mr) * 32
|
||||
self.assertEqual(unfused._cell_size, int(base * (1 + 1 / 32)))
|
||||
|
||||
|
||||
class TestHybridSWAConfigurator(CustomTestCase):
|
||||
@@ -1459,13 +1452,9 @@ class TestSWAPoolFloor(CustomTestCase):
|
||||
|
||||
|
||||
class TestFusedDraftPricing(unittest.TestCase):
|
||||
"""Fused draft KV: the boot solve must price the EXACT fused entry.
|
||||
|
||||
The fused full-side entry (host + draft + alignment pad) replaces BOTH the
|
||||
per-token full term and the per-draft-layer approximation; charging both
|
||||
would over-reserve, charging only the approximation would under-reserve
|
||||
and OOM at pool construction (the factory allocates tokens x fused entry).
|
||||
"""
|
||||
"""A fused full-side entry (host + draft + pad) replaces both the
|
||||
per-token full term and the per-draft-layer approximation in every solve:
|
||||
charging both over-reserves, charging only the approximation under-reserves."""
|
||||
|
||||
def _make(self, fused_entry, draft_layers, **runner_kwargs):
|
||||
mr = _make_model_runner(
|
||||
@@ -1500,14 +1489,6 @@ class TestFusedDraftPricing(unittest.TestCase):
|
||||
+ cfg._draft_cell_size
|
||||
)
|
||||
|
||||
def test_unfused_keeps_the_draft_layer_approximation(self):
|
||||
_, cfg = self._make(fused_entry=None, draft_layers=2)
|
||||
self.assertEqual(cfg._draft_full_layers_num, 2)
|
||||
self.assertEqual(
|
||||
cfg._cell_size,
|
||||
self._expected_cell(cfg, cfg._full_per_token * (4 + 2)),
|
||||
)
|
||||
|
||||
def test_fused_entry_replaces_full_and_draft_terms(self):
|
||||
fused_entry = 54_321
|
||||
_, cfg = self._make(fused_entry=fused_entry, draft_layers=2)
|
||||
@@ -1515,9 +1496,8 @@ class TestFusedDraftPricing(unittest.TestCase):
|
||||
self.assertEqual(cfg._cell_size, self._expected_cell(cfg, fused_entry))
|
||||
|
||||
def _assert_unified_bytes_price_the_fused_entry(self, cfg, config, available):
|
||||
"""The bytes handed to the unified pool must buy every full token at
|
||||
the fused entry -- the factory divides them by it -- and must not leave
|
||||
a full token's worth of the budget unspent."""
|
||||
"""The unified pool's bytes buy every full token at the fused entry
|
||||
(the factory divides by it) without leaving a full token unspent."""
|
||||
fused_entry = cfg._fused_full_entry
|
||||
swa_bytes = (
|
||||
config.swa_max_total_num_tokens * cfg._swa_per_token * cfg._swa_layers_num
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
import sglang.srt.model_executor.model_runner_components.cuda_graph_setup as graph_setup
|
||||
import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -31,6 +32,17 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _static_pool_translator():
|
||||
"""A static pool's translator: its plans hand every batch its own ids."""
|
||||
return KVIndexTranslator(
|
||||
req_to_token=torch.zeros((1, 8), dtype=torch.int32),
|
||||
token_to_kv_pool_allocator=None,
|
||||
token_to_kv_pool=object(),
|
||||
page_size=1,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
|
||||
class _FakeAttentionBackend:
|
||||
supports_full_cuda_graph_chunked_prefix = True
|
||||
|
||||
@@ -168,6 +180,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
prepare_dummy_forward_batch=lambda batch: prepared.append(batch) or batch,
|
||||
attn_tp_sequence_sharded=lambda _: False,
|
||||
attn_backend=attention_backend,
|
||||
kv_index_translator=_static_pool_translator(),
|
||||
)
|
||||
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
|
||||
runner.model_runner = model_runner
|
||||
@@ -321,9 +334,11 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
runner.max_context_size = None
|
||||
runner._capture_chunked_prefix = False
|
||||
runner.buffer_registry = _FakeBatchRegistry()
|
||||
translator = _static_pool_translator()
|
||||
runner.model_runner = SimpleNamespace(
|
||||
attn_tp_sequence_sharded=lambda _: False,
|
||||
prepare_dummy_forward_batch=lambda batch: batch,
|
||||
kv_index_translator=translator,
|
||||
)
|
||||
runner.enable_cp_bcg_capture = False
|
||||
runner._is_full_backend = False
|
||||
@@ -357,6 +372,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
global_forward_mode=ForwardMode.EXTEND,
|
||||
)
|
||||
translator.bind_own_plan(forward_batch)
|
||||
|
||||
static_batch = runner.load_batch(forward_batch)
|
||||
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`resolve_spec_aux_hidden_state_config` runs before the TP group exists.
|
||||
|
||||
BUG REGRESSION. `ModelRunner.init_spec_aux_hidden_state()` runs BEFORE
|
||||
`init_torch_distributed()`, so the attention-TP group does not exist while
|
||||
this resolver runs. A `get_parallel().attn_tp_size` read inside it therefore
|
||||
raises `AssertionError: attention tensor model parallel group is not
|
||||
initialized` and kills the server at startup -- for EVERY EAGLE-family run
|
||||
with a draft path, unified pool or not. It is invisible to CPU unit gates
|
||||
(they never boot a runner) and was caught only by a GPU boot guard.
|
||||
|
||||
The resolver therefore records the draft's `ModelConfig` whole, and the
|
||||
per-GPU geometry is derived at pool-build time (`draft_kv_profile`), where
|
||||
the group exists -- the same point the target divides its own heads.
|
||||
|
||||
python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py -v
|
||||
"""
|
||||
|
||||
import ast
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
_SRC = Path(_mod.__file__)
|
||||
|
||||
|
||||
def _function_named(tree, name):
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.FunctionDef) and node.name == name:
|
||||
return node
|
||||
raise AssertionError(f"{name} not found in {_SRC}")
|
||||
|
||||
|
||||
def _config(*, num_nextn_predict_layers, num_hidden_layers=1):
|
||||
return SimpleNamespace(
|
||||
num_nextn_predict_layers=num_nextn_predict_layers,
|
||||
num_hidden_layers=num_hidden_layers,
|
||||
num_attention_layers=num_hidden_layers,
|
||||
is_hybrid_swa=False,
|
||||
is_deepseek_v4_arch=False,
|
||||
)
|
||||
|
||||
|
||||
def _spec_algorithm(*, eagle: bool, eagle3: bool = False):
|
||||
return SimpleNamespace(
|
||||
is_eagle=lambda: eagle,
|
||||
is_standalone=lambda: False,
|
||||
is_eagle3=lambda: eagle3,
|
||||
)
|
||||
|
||||
|
||||
class TestResolverTouchesNoParallelState(unittest.TestCase):
|
||||
def test_no_parallel_state_in_the_eagle_resolver(self):
|
||||
"""AST pin: the EAGLE resolver runs pre-distributed-init, so any
|
||||
parallel-state read in it -- `get_parallel()` directly, or a per-GPU
|
||||
head count that divides by it -- is a boot crash, not a wrong number."""
|
||||
tree = ast.parse(_SRC.read_text())
|
||||
fn = _function_named(tree, "_resolve_eagle_aux_hidden_state")
|
||||
called = {
|
||||
n.func.id
|
||||
for n in ast.walk(fn)
|
||||
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
|
||||
}
|
||||
self.assertNotIn(
|
||||
"get_parallel",
|
||||
called,
|
||||
"_resolve_eagle_aux_hidden_state reads parallel state, but it runs "
|
||||
"before init_torch_distributed() -- the attention-TP group does not "
|
||||
"exist yet and the server dies at startup.",
|
||||
)
|
||||
methods = {
|
||||
n.func.attr
|
||||
for n in ast.walk(fn)
|
||||
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
|
||||
}
|
||||
self.assertNotIn("get_num_kv_heads", methods)
|
||||
imported = {
|
||||
alias.name
|
||||
for n in ast.walk(fn)
|
||||
if isinstance(n, ast.ImportFrom)
|
||||
for alias in n.names
|
||||
}
|
||||
self.assertNotIn("get_parallel", imported)
|
||||
|
||||
def _resolve_with(self, *, draft_path, target, draft):
|
||||
"""Run the EAGLE resolver with the config factory stubbed: `draft`
|
||||
answers the draft-mode read, `target` is the runner's own config."""
|
||||
config = _mod.SpecAuxHiddenStateConfig()
|
||||
spec = SimpleNamespace(
|
||||
speculative_draft_model_path=draft_path,
|
||||
speculative_draft_model_revision=None,
|
||||
)
|
||||
with (
|
||||
patch.object(_mod, "get_spec", return_value=spec),
|
||||
patch.object(
|
||||
_mod.ModelConfig, "from_server_args", return_value=draft
|
||||
) as factory,
|
||||
):
|
||||
_mod._resolve_eagle_aux_hidden_state(
|
||||
config=config,
|
||||
server_args=None,
|
||||
model_config=target,
|
||||
spec_algorithm=_spec_algorithm(eagle=True),
|
||||
is_draft_worker=False,
|
||||
)
|
||||
self.assertTrue(factory.call_args.kwargs["is_draft_model"])
|
||||
return config
|
||||
|
||||
def test_path_less_nextn_fuses_without_resizing_the_private_pool(self):
|
||||
"""Every `num_nextn_predict_layers = 1` assignment in model_config.py
|
||||
is guarded by `if is_draft_model`, so a path-less MTP target answers
|
||||
None for it. The fused placement must read the draft-mode config, or a
|
||||
path-less NEXTN head never fuses; the private-pool budget must keep
|
||||
reading the target, or every path-less MTP deployment's token budget
|
||||
changes."""
|
||||
target = _config(num_nextn_predict_layers=None)
|
||||
draft = _config(num_nextn_predict_layers=1)
|
||||
config = self._resolve_with(draft_path=None, target=target, draft=draft)
|
||||
self.assertIs(config.draft_model_config, draft)
|
||||
self.assertEqual(config.draft_kv_num_layers, 1)
|
||||
self.assertIsNone(config.eagle_draft_num_layers)
|
||||
|
||||
def test_draft_path_sizes_both_from_the_draft_checkpoint(self):
|
||||
target = _config(num_nextn_predict_layers=None)
|
||||
draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2)
|
||||
config = self._resolve_with(draft_path="/draft", target=target, draft=draft)
|
||||
self.assertIs(config.draft_model_config, draft)
|
||||
self.assertEqual(config.draft_kv_num_layers, 2)
|
||||
self.assertEqual(config.eagle_draft_num_layers, 2)
|
||||
|
||||
def test_the_draft_config_is_recorded_whole(self):
|
||||
"""The aux config carries the draft `ModelConfig`, never a pre-divided
|
||||
geometry: a per-GPU count recorded here would be computed without the
|
||||
TP group (every unit fixture runs at tp=1, where that is invisible)."""
|
||||
fields = _mod.SpecAuxHiddenStateConfig.__struct_fields__
|
||||
self.assertIn("draft_model_config", fields)
|
||||
self.assertNotIn("eagle_draft_total_kv_heads", fields)
|
||||
self.assertNotIn("eagle_draft_kv_head_num", fields)
|
||||
|
||||
|
||||
class TestResolverIsInertOffTheTargetEagleArm(unittest.TestCase):
|
||||
"""BUG REGRESSION. The draft-config block is guarded by "EAGLE-family
|
||||
target"; a rebase left the hybrid-SWA read outside that guard, so every
|
||||
draft worker and every spec-off runner raised `UnboundLocalError` at
|
||||
boot, and the EAGLE3 arm's try/except masked the same read into a silent
|
||||
`None`. Off the target EAGLE arm the resolver must return untouched."""
|
||||
|
||||
def _resolve(self, *, spec_algorithm, is_draft_worker):
|
||||
config = _mod.SpecAuxHiddenStateConfig()
|
||||
_mod._resolve_eagle_aux_hidden_state(
|
||||
config=config,
|
||||
server_args=None,
|
||||
model_config=None,
|
||||
spec_algorithm=spec_algorithm,
|
||||
is_draft_worker=is_draft_worker,
|
||||
)
|
||||
return config
|
||||
|
||||
def test_draft_workers_and_spec_off_runners_resolve_nothing(self):
|
||||
untouched = _mod.SpecAuxHiddenStateConfig()
|
||||
for spec_algorithm, is_draft_worker in (
|
||||
(_spec_algorithm(eagle=True), True),
|
||||
(_spec_algorithm(eagle=True, eagle3=True), True),
|
||||
(_spec_algorithm(eagle=False), False),
|
||||
):
|
||||
config = self._resolve(
|
||||
spec_algorithm=spec_algorithm, is_draft_worker=is_draft_worker
|
||||
)
|
||||
self.assertEqual(config, untouched)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Which config the EAGLE aux-hidden-state resolver sizes the draft KV from.
|
||||
|
||||
python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=3, stage="weekly", runner_config="cpu")
|
||||
|
||||
|
||||
def _config(*, num_nextn_predict_layers, num_hidden_layers=1):
|
||||
return SimpleNamespace(
|
||||
num_nextn_predict_layers=num_nextn_predict_layers,
|
||||
num_hidden_layers=num_hidden_layers,
|
||||
num_attention_layers=num_hidden_layers,
|
||||
is_hybrid_swa=False,
|
||||
is_deepseek_v4_arch=False,
|
||||
)
|
||||
|
||||
|
||||
class TestEagleResolverDraftConfig(unittest.TestCase):
|
||||
def _resolve_with(self, *, draft_path, target, draft):
|
||||
"""Run the EAGLE resolver with the config factory stubbed: `draft`
|
||||
answers the draft-mode read, `target` is the runner's own config."""
|
||||
config = _mod.SpecAuxHiddenStateConfig()
|
||||
spec = SimpleNamespace(
|
||||
speculative_draft_model_path=draft_path,
|
||||
speculative_draft_model_revision=None,
|
||||
)
|
||||
with (
|
||||
patch.object(_mod, "get_spec", return_value=spec),
|
||||
patch.object(
|
||||
_mod.ModelConfig, "from_server_args", return_value=draft
|
||||
) as factory,
|
||||
):
|
||||
_mod._resolve_eagle_aux_hidden_state(
|
||||
config=config,
|
||||
server_args=None,
|
||||
model_config=target,
|
||||
spec_algorithm=SimpleNamespace(
|
||||
is_eagle=lambda: True,
|
||||
is_standalone=lambda: False,
|
||||
is_eagle3=lambda: False,
|
||||
),
|
||||
is_draft_worker=False,
|
||||
)
|
||||
self.assertTrue(factory.call_args.kwargs["is_draft_model"])
|
||||
return config
|
||||
|
||||
def test_path_less_nextn_fuses_without_resizing_the_private_pool(self):
|
||||
"""A path-less MTP head sizes its fused draft KV from the draft-mode
|
||||
config (the only one that names its NEXTN layers), while the private
|
||||
draft pool keeps reading the target config, so its token budget is
|
||||
unchanged."""
|
||||
target = _config(num_nextn_predict_layers=None)
|
||||
draft = _config(num_nextn_predict_layers=1)
|
||||
config = self._resolve_with(draft_path=None, target=target, draft=draft)
|
||||
self.assertIs(config.draft_model_config, draft)
|
||||
self.assertEqual(config.draft_kv_num_layers, 1)
|
||||
self.assertIsNone(config.eagle_draft_num_layers)
|
||||
|
||||
def test_draft_path_sizes_both_from_the_draft_checkpoint(self):
|
||||
target = _config(num_nextn_predict_layers=None)
|
||||
draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2)
|
||||
config = self._resolve_with(draft_path="/draft", target=target, draft=draft)
|
||||
self.assertIs(config.draft_model_config, draft)
|
||||
self.assertEqual(config.draft_kv_num_layers, 2)
|
||||
self.assertEqual(config.eagle_draft_num_layers, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -13,11 +13,12 @@
|
||||
# ==============================================================================
|
||||
"""ForwardBatch construction wires the unified write-loc rebind.
|
||||
|
||||
`init_new` must call `kv_index_translator.rebind_write_loc`: a construction
|
||||
path that skips it ships VIRTUAL write ids to the kernels, a silent
|
||||
wrong-slot store. Also runs the REAL `_pad_inputs_to_size` against a live
|
||||
translator, since pad lanes are zeros and zeros must derive to the slot-0
|
||||
sink. Sliding-window semantics are pinned in test_kv_index_translator.py.
|
||||
`init_new` must bind the batch to a plan (`translator.own_plan`,
|
||||
`kv_loc_plan.bind`): a construction path that skips it ships VIRTUAL write
|
||||
ids to the kernels, a silent wrong-slot store. Also runs the REAL
|
||||
`_pad_inputs_to_size` against a live translator, since pad lanes are zeros
|
||||
and must write the slot-0 sink in every id space. Sliding-window semantics
|
||||
are pinned in test_kv_index_translator.py.
|
||||
|
||||
python -m pytest test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py -v
|
||||
"""
|
||||
@@ -32,6 +33,7 @@ from unittest.mock import create_autospec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -70,22 +72,25 @@ def _armed_source(v2p, swa_map):
|
||||
)
|
||||
src.is_translating = True
|
||||
src._translate_full = lambda t, out=None: v2p[t.to(torch.int64)]
|
||||
# The WRITE loc has its own translate because under DCP it arrives widened;
|
||||
# at dcp_size == 1 it is the read translate, so arm it with the same fake.
|
||||
src._translate_write_full = src._translate_full
|
||||
# Phase 2 derives from physical values through p2v + the swa v2p; arm the
|
||||
# inverse of the fake v2p (ps=1, so the expected swa loc for virtual t is
|
||||
# swa_map[t]).
|
||||
p2v = torch.zeros(int(v2p.max()) + 1, dtype=torch.int64)
|
||||
p2v[v2p] = torch.arange(v2p.numel(), dtype=torch.int64)
|
||||
src._full_p2v_table = p2v
|
||||
src._swa_v2p_table = swa_map
|
||||
src._spaces = {
|
||||
# The WRITE loc has its own translate because under DCP it arrives
|
||||
# widened; at dcp_size == 1 it is the read translate.
|
||||
IdSpaceKind.FULL: IdSpace(
|
||||
key=(IdSpaceKind.FULL, "test"), write=src._translate_full
|
||||
),
|
||||
# The sliding-window ids derive from the virtual window (ps=1, so the
|
||||
# swa id for virtual t is swa_map[t]).
|
||||
IdSpaceKind.SLIDING_WINDOW: IdSpace(
|
||||
key=(IdSpaceKind.SLIDING_WINDOW, "test"),
|
||||
write=lambda t: swa_map[t.to(torch.int64)],
|
||||
),
|
||||
}
|
||||
return src
|
||||
|
||||
|
||||
def _call_names(func) -> list:
|
||||
"""Dotted call targets appearing in `func`'s body, e.g.
|
||||
'model_runner.kv_index_translator.rebind_write_loc'."""
|
||||
'kv_loc_plan.bind'."""
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(func)))
|
||||
names = []
|
||||
for node in ast.walk(tree):
|
||||
@@ -104,13 +109,15 @@ def _call_names(func) -> list:
|
||||
class TestForwardBatchWiring(CustomTestCase):
|
||||
"""Critical-path bookkeeping: the construction-time call sites."""
|
||||
|
||||
def test_init_new_calls_the_rebind(self):
|
||||
self.assertIn(
|
||||
"model_runner.kv_index_translator.rebind_write_loc",
|
||||
_call_names(ForwardBatch.init_new.__func__),
|
||||
"init_new must rebind the write loc through the source; a batch "
|
||||
"built without it ships virtual ids to the kernels",
|
||||
)
|
||||
def test_init_new_binds_the_plan(self):
|
||||
names = _call_names(ForwardBatch.init_new.__func__)
|
||||
for call in ("translator.own_plan", "kv_loc_plan.bind"):
|
||||
self.assertIn(
|
||||
call,
|
||||
names,
|
||||
"init_new must take its write ids from a plan; a batch built "
|
||||
"without one ships virtual ids to the kernels",
|
||||
)
|
||||
|
||||
|
||||
class TestPadComposesWithDerivation(CustomTestCase):
|
||||
@@ -120,11 +127,11 @@ class TestPadComposesWithDerivation(CustomTestCase):
|
||||
kv_index_translator=src,
|
||||
)
|
||||
|
||||
def test_pad_lanes_derive_to_sink_and_slices_stay_pointwise(self):
|
||||
"""The REAL `_pad_inputs_to_size` composes with phase 2: pad lanes are
|
||||
zeros, zeros derive to the slot-0 sink, and any slice of the padded
|
||||
tensor (the TBO-child shape) derives pointwise -- no handover call
|
||||
exists for the pad to make."""
|
||||
def test_pad_lanes_write_the_sink_in_both_spaces(self):
|
||||
"""The REAL `_pad_inputs_to_size` pads the batch's write ids, and the
|
||||
sliding-window ids its plan hands out follow them: pad lanes write the
|
||||
slot-0 sink in both spaces, and both stay the same length as the
|
||||
batch's tokens."""
|
||||
n, padded = 3, 6
|
||||
v2p = torch.arange(64, dtype=torch.int64) * 3
|
||||
swa_map = torch.arange(64, dtype=torch.int64) * 5
|
||||
@@ -133,36 +140,37 @@ class TestPadComposesWithDerivation(CustomTestCase):
|
||||
fb = _make_fb(virt.clone())
|
||||
fb.positions = torch.arange(n, dtype=torch.int64)
|
||||
fb.lora_ids = [None] * fb.batch_size
|
||||
src.rebind_write_loc(fb)
|
||||
src.bind_own_plan(fb)
|
||||
swa = IdSpaceKind.SLIDING_WINDOW
|
||||
self.assertTrue(torch.equal(fb.out_cache_loc, v2p[virt]))
|
||||
self.assertTrue(torch.equal(src.write_ids(fb, swa), swa_map[virt]))
|
||||
|
||||
fb._pad_inputs_to_size(self._fake_runner_for_pad(src), padded, fb.batch_size)
|
||||
|
||||
self.assertEqual(fb.out_cache_loc.shape[0], padded)
|
||||
# Padded tail lanes go to slot 0 -- the reserved dummy-write sink.
|
||||
self.assertTrue(bool((fb.out_cache_loc[n:] == 0).all()))
|
||||
loc = src._swa_write_loc_unified(fb.out_cache_loc)
|
||||
self.assertTrue(torch.equal(loc[:n], swa_map[virt]))
|
||||
self.assertTrue(bool((loc[n:] == 0).all()))
|
||||
self.assertEqual(loc.dtype, torch.int64)
|
||||
# The TBO-child shape: a slice of the PADDED tensor derives pointwise.
|
||||
sub = src._swa_write_loc_unified(fb.out_cache_loc[1:5])
|
||||
self.assertTrue(torch.equal(sub, loc[1:5]))
|
||||
for loc, want in (
|
||||
(fb.out_cache_loc, v2p[virt]),
|
||||
(src.write_ids(fb, swa), swa_map[virt]),
|
||||
):
|
||||
self.assertEqual(loc.shape[0], padded)
|
||||
self.assertTrue(torch.equal(loc[:n], want))
|
||||
self.assertTrue(bool((loc[n:] == 0).all()))
|
||||
self.assertEqual(loc.dtype, torch.int64)
|
||||
|
||||
def test_empty_loc_rebinds_to_empty(self):
|
||||
src = _armed_source(
|
||||
torch.arange(8, dtype=torch.int64), torch.arange(8, dtype=torch.int64)
|
||||
)
|
||||
fb = _make_fb(torch.empty(0, dtype=torch.int64))
|
||||
src.rebind_write_loc(fb)
|
||||
src.bind_own_plan(fb)
|
||||
self.assertEqual(fb.out_cache_loc.numel(), 0)
|
||||
self.assertEqual(src._swa_write_loc_unified(fb.out_cache_loc).numel(), 0)
|
||||
self.assertEqual(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW).numel(), 0)
|
||||
|
||||
|
||||
class TestReadRailTranslatesAtProduction(CustomTestCase):
|
||||
"""The model-door READ indices (req_to_token-derived, VIRTUAL under the
|
||||
unified pool) are translated at their PRODUCTION site -- the cache then
|
||||
holds the physical result and the pool door never translates."""
|
||||
"""The model-door READ indices are physical from their PRODUCTION site
|
||||
(a gather from the iteration's plan under the unified pool) -- the cache
|
||||
then holds the physical result and the pool door never translates."""
|
||||
|
||||
def _fb_for_one_shot(self):
|
||||
fb = _make_fb(torch.tensor([1, 2], dtype=torch.int64))
|
||||
@@ -172,39 +180,84 @@ class TestReadRailTranslatesAtProduction(CustomTestCase):
|
||||
fb.req_pool_indices = torch.tensor([0, 1], dtype=torch.int64)
|
||||
return fb
|
||||
|
||||
def test_one_shot_indices_translated_once_and_cached(self):
|
||||
def test_one_shot_indices_gathered_from_the_plan_once_and_cached(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix
|
||||
|
||||
plans = []
|
||||
|
||||
def pack(plan, *, req_pool_indices, seq_lens, indptr, out, **_):
|
||||
plans.append(plan)
|
||||
out.fill_(7)
|
||||
return True
|
||||
|
||||
fb = self._fb_for_one_shot()
|
||||
fb.kv_loc_plan = object()
|
||||
# autospec, not a bare namespace: setting a name the translator does
|
||||
# not have raises, so renaming the method breaks this test loudly.
|
||||
fake_translator = create_autospec(KVIndexTranslator, instance=True)
|
||||
fake_translator.pack_read_stream = pack
|
||||
fake_backend = SimpleNamespace(kv_index_translator=fake_translator)
|
||||
with patch.object(mix, "get_attn_backend", return_value=fake_backend):
|
||||
r1 = fb.fetch_mha_one_shot_kv_indices()
|
||||
r2 = fb.fetch_mha_one_shot_kv_indices()
|
||||
|
||||
# One gather, from the iteration's plan; the cache holds its result.
|
||||
self.assertEqual(plans, [fb.kv_loc_plan])
|
||||
self.assertIs(r2, r1)
|
||||
self.assertTrue(torch.equal(r1, torch.full((5,), 7, dtype=torch.int32)))
|
||||
|
||||
def _fb_for_chunks(self):
|
||||
fb = self._fb_for_one_shot()
|
||||
fb.kv_loc_plan = object()
|
||||
fb.num_prefix_chunks = 2
|
||||
fb.prefix_chunk_starts = [torch.tensor([0, 0]), torch.tensor([2, 1])]
|
||||
fb.prefix_chunk_seq_lens = [torch.tensor([2, 1]), torch.tensor([1, 2])]
|
||||
fb.prefix_chunk_cu_seq_lens = [
|
||||
torch.tensor([0, 2, 3], dtype=torch.int32),
|
||||
torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
]
|
||||
fb.prefix_chunk_num_tokens = [3, 3]
|
||||
fb.prefix_chunk_starts_cpu = [[0, 0], [2, 1]]
|
||||
fb.prefix_chunk_seq_lens_cpu = [[2, 1], [1, 2]]
|
||||
return fb
|
||||
|
||||
def test_chunk_indices_gathered_from_the_plan(self):
|
||||
"""On a translating pool each prefix chunk's ids come out of the plan's
|
||||
table: one gather per chunk, at the chunk's start, no translate."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix
|
||||
|
||||
calls = []
|
||||
sentinel = torch.arange(5, dtype=torch.int64) + 5000
|
||||
|
||||
def translate(t):
|
||||
calls.append(t)
|
||||
return sentinel
|
||||
def pack(plan, *, req_pool_indices, seq_lens, indptr, out, kv_start_idx):
|
||||
calls.append((plan, seq_lens, indptr, kv_start_idx))
|
||||
out.fill_(9)
|
||||
return True
|
||||
|
||||
fb = self._fb_for_one_shot()
|
||||
fake_pool = SimpleNamespace(
|
||||
req_to_token=torch.zeros((4, 16), dtype=torch.int32)
|
||||
)
|
||||
# autospec, not a bare namespace: setting a name the translator does
|
||||
# not have raises, so renaming the method breaks this test loudly.
|
||||
fb = self._fb_for_chunks()
|
||||
fake_translator = create_autospec(KVIndexTranslator, instance=True)
|
||||
fake_translator.translate_full_attn_ids = translate
|
||||
fake_translator.reads_are_translated = True
|
||||
fake_translator.pack_read_stream = pack
|
||||
fake_backend = SimpleNamespace(kv_index_translator=fake_translator)
|
||||
fake_pool = SimpleNamespace(req_to_token=torch.zeros((4, 8), dtype=torch.int32))
|
||||
with (
|
||||
patch.object(mix, "get_req_to_token_pool", return_value=fake_pool),
|
||||
patch.object(mix, "get_attn_backend", return_value=fake_backend),
|
||||
patch.object(mix, "create_flashinfer_kv_indices_triton"),
|
||||
patch.object(mix, "get_req_to_token_pool", return_value=fake_pool),
|
||||
):
|
||||
r1 = fb.fetch_mha_one_shot_kv_indices()
|
||||
r2 = fb.fetch_mha_one_shot_kv_indices()
|
||||
fb.prepare_chunked_kv_indices(torch.device("cpu"))
|
||||
|
||||
self.assertIs(r1, sentinel) # production site translated
|
||||
self.assertIs(r2, sentinel) # cache holds the TRANSLATED result
|
||||
self.assertEqual(len(calls), 1) # translated exactly once
|
||||
self.assertEqual(calls[0].dtype, torch.int32) # raw producer output
|
||||
self.assertEqual(len(calls), 2)
|
||||
for idx, (plan, seq_lens, indptr, kv_start_idx) in enumerate(calls):
|
||||
self.assertIs(plan, fb.kv_loc_plan)
|
||||
self.assertIs(seq_lens, fb.prefix_chunk_seq_lens[idx])
|
||||
self.assertIs(indptr, fb.prefix_chunk_cu_seq_lens[idx])
|
||||
self.assertIs(kv_start_idx, fb.prefix_chunk_starts[idx])
|
||||
fake_translator.translate_dcp_read_ids.assert_not_called()
|
||||
for chunk in fb.prefix_chunk_kv_indices:
|
||||
self.assertTrue(bool((chunk == 9).all()))
|
||||
|
||||
def test_one_shot_indices_noop_on_unmigrated_backend(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -11,34 +11,11 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""`--enable-unified-memory` speculative-decoding allow-list.
|
||||
"""`--enable-unified-memory` speculative-decoding backend allow-lists.
|
||||
|
||||
Three audited arms (see `handle_unified_memory_pool`), each with its own
|
||||
backend constraints:
|
||||
* DSPARK: the target verifies on the audited verify set (`triton` /
|
||||
`trtllm_mla` / `cutedsl_mla` / `tokenspeed_mla` / `flashmla` /
|
||||
`flashinfer` / `fa3`, which includes the MLA family). Its draft fuses
|
||||
into the target's pages when the fused-draft decision allows, else keeps
|
||||
a private pool, so the gate leaves the draft backend unconstrained.
|
||||
* DFLASH: the target verifies on `triton` / `fa3` / `flashinfer`; the MLA
|
||||
verify family must not leak into this arm.
|
||||
* EAGLE/EAGLE3: unified targets only (hybrid-SWA or mamba hybrids, either
|
||||
full-pool kind) -- the draft's KV lives fused inside the full pool's
|
||||
page envelope (`DenseDraftRegion`), and a draft that does not fuse is
|
||||
refused at boot. The target's verify set follows the
|
||||
host kind: the audited verify set on an MLA host, `triton` /
|
||||
`flashinfer` / `fa3` on an MHA host, and an unresolved backend is
|
||||
refused. The draft worker (its backend resolves separately: explicit
|
||||
flag first, else it inherits the target's) runs on `triton` /
|
||||
`flashinfer` / `fa3`.
|
||||
|
||||
Every arm drafts a linear chain (`--speculative-eagle-topk` in {None, 1}).
|
||||
Everything else (NGRAM / STANDALONE / registered customs) stays refused.
|
||||
NGRAM's refusal is load-bearing, not pending: like tree verify it relocates
|
||||
accepted tokens one at a time inside the target pool, which the unified
|
||||
pool's page-granular `move_kv_cache` cannot express. Pinned so no arm silently
|
||||
widens to an unaudited algorithm, family, tree shape, or backend -- and so one
|
||||
arm's addition never perturbs another.
|
||||
A speculative backend that does not read through the KV-index translator
|
||||
reads the unified pool with virtual ids and produces wrong tokens without
|
||||
crashing, so each arm's admitted backend set is pinned exactly.
|
||||
|
||||
python -m pytest test/registered/unit/server_args/test_unified_memory_spec_gate.py -v
|
||||
"""
|
||||
@@ -55,14 +32,13 @@ from sglang.srt.configs.model_config import AttentionArch
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _PlainHFConfig:
|
||||
"""Matches none of the hybrid-arch isinstance probes."""
|
||||
"""Matches none of the hybrid-arch probes (`mambaish_config` screens on
|
||||
the architecture name)."""
|
||||
|
||||
# `mambaish_config` screens on the architecture name, so a stub that
|
||||
# declares none is what "not a hybrid arch" looks like to it.
|
||||
architectures = ["LlamaForCausalLM"]
|
||||
|
||||
def get_text_config(self):
|
||||
@@ -77,18 +53,10 @@ def _accepts(
|
||||
backend: str | None = "triton",
|
||||
draft_backend: str | None = None,
|
||||
dcp_size: int = 1,
|
||||
retraction_backup: str | None = None,
|
||||
fields: dict | None = None,
|
||||
attention_arch: AttentionArch = AttentionArch.MHA,
|
||||
) -> bool:
|
||||
"""Run just `handle_unified_memory_pool` against a minimal stand-in.
|
||||
|
||||
ServerArgs' real constructor pulls in a model config; this exercises the
|
||||
single handler under test with the fields it reads (disaggregation off, so
|
||||
only the speculative / cache / dcp / cuda-graph checks run). ServerArgs is
|
||||
a msgspec Struct, so a field is restated with `force_setattr`, which the
|
||||
Struct's own `__setattr__` guard allows and `object.__setattr__` does not.
|
||||
"""
|
||||
"""Run just `handle_unified_memory_pool` on a ServerArgs whose fields are
|
||||
restated with `force_setattr` (ServerArgs is a msgspec Struct)."""
|
||||
sa = ServerArgs(model_path="dummy")
|
||||
for name, value in {
|
||||
"enable_unified_memory": True,
|
||||
@@ -98,13 +66,12 @@ def _accepts(
|
||||
"speculative_draft_attention_backend": draft_backend,
|
||||
"enable_hierarchical_cache": False,
|
||||
"enable_lmcache": False,
|
||||
"disaggregation_decode_retraction_backup": retraction_backup,
|
||||
"disaggregation_decode_retraction_backup": None,
|
||||
"dcp_size": dcp_size,
|
||||
"cuda_graph_config": None,
|
||||
"attention_backend": backend,
|
||||
"prefill_attention_backend": None,
|
||||
"decode_attention_backend": None,
|
||||
**(fields or {}),
|
||||
}.items():
|
||||
msgspec.structs.force_setattr(sa, name, value)
|
||||
sa._model_config = SimpleNamespace(
|
||||
@@ -120,298 +87,103 @@ def _accepts(
|
||||
return False
|
||||
|
||||
|
||||
# The audited verify set (includes the MLA family).
|
||||
_VERIFY_BACKENDS = (
|
||||
"triton",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
"flashinfer",
|
||||
"fa3",
|
||||
)
|
||||
_MHA_RAILS = ("triton", "flashinfer", "fa3")
|
||||
|
||||
|
||||
class TestUnifiedMemorySpecGate(unittest.TestCase):
|
||||
# Verify-audited backends for the DSPARK arm (includes the MLA family).
|
||||
DSPARK_BACKENDS = (
|
||||
"triton",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
"flashinfer",
|
||||
"fa3",
|
||||
)
|
||||
# The translated MHA rails: the EAGLE and DFLASH arms' target backends
|
||||
# on an MHA host, and every fused draft's backend.
|
||||
MHA_RAILS = ("triton", "flashinfer", "fa3")
|
||||
# Algorithms with no audited unified-pool verify rails.
|
||||
# "NEXTN" is deliberately absent: the CLI alias collapses it to
|
||||
# "EAGLE" in handle_speculative_decoding BEFORE this gate runs
|
||||
# (arg_groups/pipeline.py orders the hooks), so the raw string can
|
||||
# never reach the gate -- a case on it would test an impossible
|
||||
# input. The aliased spelling is covered by the EAGLE cases.
|
||||
UNAUDITED_ALGORITHMS = ("NGRAM", "STANDALONE")
|
||||
VERIFY_BACKENDS = _VERIFY_BACKENDS
|
||||
MHA_RAILS = _MHA_RAILS
|
||||
MLA_ONLY = tuple(b for b in _VERIFY_BACKENDS if b not in _MHA_RAILS)
|
||||
UNAUDITED = ("fa4", "trtllm_mha")
|
||||
|
||||
def test_dspark_admitted_on_every_audited_backend(self):
|
||||
"""Each entry is a claim that the backend's spec verify path
|
||||
translates. Adding one here without the code is how a unified run
|
||||
silently reads the pool with virtual ids."""
|
||||
for backend in self.DSPARK_BACKENDS:
|
||||
self.assertTrue(
|
||||
_accepts("DSPARK", backend=backend),
|
||||
f"DSPARK should pass on verify-audited backend {backend}",
|
||||
)
|
||||
|
||||
def test_dspark_refused_on_unaudited_backends(self):
|
||||
"""fa4 and trtllm_mha have no translated spec verify path."""
|
||||
for backend in ("fa4", "trtllm_mha"):
|
||||
self.assertFalse(_accepts("DSPARK", backend=backend))
|
||||
|
||||
def test_tree_drafting_refused(self):
|
||||
"""Tree verify needs a per-token move inside the TARGET pool, which
|
||||
the unified pool's page-granular `move_kv_cache` cannot express.
|
||||
DSPARK demonstrates the rule here; the check itself is unconditional,
|
||||
so it holds for every admitted arm."""
|
||||
for topk in (2, 4, 8):
|
||||
self.assertFalse(_accepts("DSPARK", topk=topk))
|
||||
# The shape knob must not disturb spec-off.
|
||||
self.assertTrue(_accepts(None, topk=None))
|
||||
|
||||
def test_dspark_draft_backend_is_not_constrained(self):
|
||||
"""The gate does not constrain DSPARK's draft backend: a draft whose
|
||||
backend is off the translated rails keeps a private pool (the
|
||||
fused-draft decision declines fusion), where its translator is a
|
||||
passthrough and any backend reads correctly. Model hooks declare one
|
||||
on the operator's behalf -- Kimi-Linear / Kimi-K3 + DSPARK on SM100
|
||||
declares `trtllm_mha` -- so refusing it would refuse a configuration
|
||||
the operator never touched."""
|
||||
for draft_backend in (None, *self.DSPARK_BACKENDS, "fa4", "trtllm_mha"):
|
||||
self.assertTrue(
|
||||
_accepts("DSPARK", draft_backend=draft_backend),
|
||||
f"DSPARK draft backend {draft_backend} should pass",
|
||||
)
|
||||
def test_dspark_verify_backends(self):
|
||||
for backend in self.VERIFY_BACKENDS:
|
||||
self.assertTrue(_accepts("DSPARK", backend=backend), backend)
|
||||
for backend in self.UNAUDITED:
|
||||
self.assertFalse(_accepts("DSPARK", backend=backend), backend)
|
||||
|
||||
def test_dcp_keeps_flashinfer_off_the_verify_list(self):
|
||||
"""Under --dcp-size > 1 the read ids stay DCP-widened for the consumer
|
||||
to finish; flashinfer's spec verify gathers its CSR args with no DCP
|
||||
read translation, so it is refused there while the MLA verify family,
|
||||
which builds its own DCP block table, still passes."""
|
||||
self.assertTrue(
|
||||
_accepts(
|
||||
"DSPARK",
|
||||
backend="trtllm_mla",
|
||||
dcp_size=2,
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
_accepts(
|
||||
"DSPARK",
|
||||
backend="flashinfer",
|
||||
dcp_size=2,
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
)
|
||||
)
|
||||
# Without DCP flashinfer stays admitted.
|
||||
self.assertTrue(
|
||||
_accepts(
|
||||
"DSPARK",
|
||||
backend="flashinfer",
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
)
|
||||
)
|
||||
"""Under DCP flashinfer's spec verify reads DCP-widened ids; the MLA
|
||||
verify family builds its own DCP block table."""
|
||||
mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA)
|
||||
self.assertTrue(_accepts("DSPARK", backend="trtllm_mla", dcp_size=2, **mla))
|
||||
self.assertFalse(_accepts("DSPARK", backend="flashinfer", dcp_size=2, **mla))
|
||||
self.assertTrue(_accepts("DSPARK", backend="flashinfer", **mla))
|
||||
|
||||
def test_dflash_admitted_only_on_its_verify_backends(self):
|
||||
"""DFLASH's target verifies on triton / fa3 / flashinfer. fa4 and
|
||||
trtllm_mha have no translated spec verify path, and the MLA verify
|
||||
family the DSPARK arm admits must not leak into this arm."""
|
||||
def test_dflash_verify_backends(self):
|
||||
"""DFLASH verifies on the MHA rails only; the MLA family DSPARK admits
|
||||
must not leak into this arm."""
|
||||
for backend in self.MHA_RAILS:
|
||||
self.assertTrue(
|
||||
_accepts("DFLASH", backend=backend),
|
||||
f"DFLASH should pass on verify-audited backend {backend}",
|
||||
)
|
||||
for backend in ("fa4", "trtllm_mha") + tuple(
|
||||
b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS
|
||||
):
|
||||
self.assertFalse(
|
||||
_accepts("DFLASH", backend=backend),
|
||||
f"DFLASH must be refused on backend {backend}",
|
||||
)
|
||||
self.assertTrue(_accepts("DFLASH", backend=backend), backend)
|
||||
for backend in self.UNAUDITED + self.MLA_ONLY:
|
||||
self.assertFalse(_accepts("DFLASH", backend=backend), backend)
|
||||
|
||||
def test_eagle_family_admitted_on_hybrid_swa(self):
|
||||
"""EAGLE/EAGLE3 chain on a hybrid-SWA target with audited verify
|
||||
backends is the fused-draft-KV configuration -- both spellings,
|
||||
resolved or unset topk, every audited backend."""
|
||||
def test_eagle_verify_backends_on_an_mha_host(self):
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
for topk in (None, 1):
|
||||
for backend in self.MHA_RAILS:
|
||||
self.assertTrue(
|
||||
_accepts(algorithm, topk=topk, backend=backend),
|
||||
f"{algorithm} topk={topk} backend={backend} should "
|
||||
"pass on a hybrid-SWA target",
|
||||
f"{algorithm} topk={topk} {backend}",
|
||||
)
|
||||
|
||||
def test_eagle_refused_on_dense_targets(self):
|
||||
"""No fused draft region outside the unified composites: a dense
|
||||
(non-hybrid) target must be refused, not fail at boot."""
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
self.assertFalse(_accepts(algorithm, is_hybrid_swa=False))
|
||||
|
||||
def test_eagle_admitted_on_mamba_mha(self):
|
||||
"""A mamba hybrid off the MLA backend provisions the fused draft
|
||||
region in its MHA full sub-pool; refusing it here strands the whole
|
||||
mamba x EAGLE matrix."""
|
||||
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
self.assertTrue(_accepts(algorithm, is_hybrid_swa=False))
|
||||
|
||||
def test_eagle_on_mla_mamba_verifies_on_the_mla_family(self):
|
||||
"""An MLA mamba hybrid fuses into MLA pages, so the target verifies on
|
||||
the MLA backend family, while the fused draft is MHA-shaped and must run
|
||||
on the translated MHA rails: an MLA-only backend is refused for the
|
||||
draft whether named or inherited."""
|
||||
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
|
||||
for backend in self.DSPARK_BACKENDS:
|
||||
self.assertTrue(
|
||||
_accepts(
|
||||
"EAGLE",
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
backend=backend,
|
||||
draft_backend="triton",
|
||||
),
|
||||
f"EAGLE on an MLA host should pass on {backend}",
|
||||
for backend in self.UNAUDITED + self.MLA_ONLY:
|
||||
self.assertFalse(
|
||||
_accepts(algorithm, backend=backend, draft_backend="triton"),
|
||||
f"{algorithm} {backend}",
|
||||
)
|
||||
|
||||
def test_eagle_draft_backend(self):
|
||||
"""An unset draft backend inherits the target's; an explicit one must
|
||||
be on the MHA rails."""
|
||||
self.assertTrue(_accepts("EAGLE", draft_backend=None))
|
||||
for draft_backend in self.MHA_RAILS:
|
||||
self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend))
|
||||
for draft_backend in self.UNAUDITED:
|
||||
self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend))
|
||||
|
||||
def test_eagle_on_an_mla_host(self):
|
||||
"""An MLA mamba hybrid verifies on the audited verify set, while its
|
||||
fused draft is MHA-shaped: an MLA-only draft backend is refused whether
|
||||
named or inherited."""
|
||||
mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA)
|
||||
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
|
||||
for backend in self.VERIFY_BACKENDS:
|
||||
self.assertTrue(
|
||||
_accepts("EAGLE", backend=backend, draft_backend="triton", **mla),
|
||||
backend,
|
||||
)
|
||||
self.assertFalse(
|
||||
_accepts("EAGLE", backend="fa4", draft_backend="triton", **mla)
|
||||
)
|
||||
for backend in ("trtllm_mla", "flashmla"):
|
||||
for draft_backend in (None, backend):
|
||||
self.assertFalse(
|
||||
_accepts(
|
||||
"EAGLE",
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
backend=backend,
|
||||
draft_backend=draft_backend,
|
||||
"EAGLE", backend=backend, draft_backend=draft_backend, **mla
|
||||
),
|
||||
f"MLA-only draft backend {draft_backend or backend}",
|
||||
draft_backend or backend,
|
||||
)
|
||||
for backend in (None, "fa4"):
|
||||
self.assertFalse(
|
||||
_accepts(
|
||||
"EAGLE",
|
||||
is_hybrid_swa=False,
|
||||
attention_arch=AttentionArch.MLA,
|
||||
backend=backend,
|
||||
)
|
||||
)
|
||||
|
||||
def test_eagle_refused_tree_topk(self):
|
||||
"""Tree verify is not audited for the unified pool; only a linear
|
||||
chain passes."""
|
||||
for topk in (2, 4, 8):
|
||||
self.assertFalse(_accepts("EAGLE", topk=topk))
|
||||
|
||||
def test_eagle_refused_unaudited_backends(self):
|
||||
"""Unaudited backends stay out, and the MLA verify set from the
|
||||
DSPARK arm must not leak into the EAGLE arm."""
|
||||
for backend in ("fa4", "trtllm_mha") + tuple(
|
||||
b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS
|
||||
):
|
||||
self.assertFalse(_accepts("EAGLE", backend=backend))
|
||||
|
||||
def test_eagle_refused_unset_backend(self):
|
||||
"""A real boot resolves the default backend before this gate runs, so
|
||||
an unset one here is unresolved: the gate refuses it rather than
|
||||
trusting it."""
|
||||
self.assertFalse(_accepts("EAGLE", backend=None))
|
||||
|
||||
def test_eagle_draft_backend_pinned(self):
|
||||
"""The draft worker resolves its own backend: unset inherits the
|
||||
target's (triton here), an explicit triton / flashinfer / fa3 passes,
|
||||
and anything else refuses."""
|
||||
self.assertTrue(_accepts("EAGLE", draft_backend=None))
|
||||
for draft_backend in self.MHA_RAILS:
|
||||
self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend))
|
||||
for draft_backend in ("fa4", "trtllm_mha"):
|
||||
self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend))
|
||||
|
||||
def test_hierarchical_cache_refused_with_an_eagle_draft(self):
|
||||
"""Under HiCache an EAGLE draft keeps a private pool, and an MTP
|
||||
head's KV would be packed into the target's page-envelope host pool,
|
||||
which refuses per-layer draft loads. Without a draft HiCache is not
|
||||
this gate's to refuse, and with DSPARK HiCache declines draft fusion,
|
||||
so the draft keeps its private pool."""
|
||||
hicache = {"enable_hierarchical_cache": True}
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
self.assertFalse(_accepts(algorithm, fields=hicache))
|
||||
self.assertTrue(_accepts(None, fields=hicache))
|
||||
self.assertTrue(_accepts("DSPARK", fields=hicache))
|
||||
|
||||
def test_pd_decode_refusals_raise_their_own_assertions(self):
|
||||
"""The PD-decode branches screen the model with `mambaish_config` and
|
||||
refuse with their own assertion, not an error from the screen."""
|
||||
pd_decode = {
|
||||
"disaggregation_mode": "decode",
|
||||
"disaggregation_transfer_backend": "mooncake",
|
||||
"pp_size": 1,
|
||||
"enable_hisparse": False,
|
||||
}
|
||||
for extra in (
|
||||
{
|
||||
"disaggregation_decode_enable_offload_kvcache": True,
|
||||
"hicache_storage_backend": "file",
|
||||
},
|
||||
{"disaggregation_decode_retraction_backup": "host_pool"},
|
||||
):
|
||||
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
|
||||
self.assertFalse(
|
||||
_accepts(None, is_hybrid_swa=False, fields={**pd_decode, **extra})
|
||||
)
|
||||
|
||||
def test_host_pool_retraction_refused_with_an_eagle_draft(self):
|
||||
"""The host-pool retraction backup builds the draft's host pool off
|
||||
its device pool, which an EAGLE draft, always fused on the unified
|
||||
pool, does not have. Without a draft the backup is not this gate's to
|
||||
refuse, and with DFLASH or DSPARK it declines draft fusion instead."""
|
||||
for algorithm in ("EAGLE", "EAGLE3"):
|
||||
with self.subTest(algorithm=algorithm):
|
||||
self.assertTrue(_accepts(algorithm))
|
||||
self.assertFalse(_accepts(algorithm, retraction_backup="host_pool"))
|
||||
for algorithm in (None, "DFLASH", "DSPARK"):
|
||||
with self.subTest(algorithm=algorithm):
|
||||
self.assertTrue(_accepts(algorithm, retraction_backup="host_pool"))
|
||||
|
||||
def test_dspark_arm_unchanged(self):
|
||||
"""The EAGLE addition must not perturb DSPARK: its MLA verify set
|
||||
passes, its chain constraint holds, and unaudited backends refuse."""
|
||||
for backend in self.DSPARK_BACKENDS:
|
||||
self.assertTrue(
|
||||
_accepts("DSPARK", backend=backend),
|
||||
f"DSPARK should pass on verify-audited backend {backend}",
|
||||
)
|
||||
self.assertFalse(_accepts("DSPARK", backend="fa4"))
|
||||
self.assertFalse(_accepts("DSPARK", topk=4))
|
||||
|
||||
def test_ngram_refused_because_it_relocates_target_kv_per_token(self):
|
||||
"""BUG REGRESSION. NGRAM was admitted on the reasoning that
|
||||
it carries no draft KV, so only the target-verify rails matter. But a
|
||||
tree-shaped drafter finalizes a batch through
|
||||
`move_accept_tokens_to_target_kvcache`, which relocates INDIVIDUAL
|
||||
accepted tokens inside the target pool -- and the unified pool's
|
||||
`move_kv_cache` is compaction-only (whole page envelopes). Every
|
||||
unified NGRAM cell died there: a reshape past the end on the MHA/mamba
|
||||
hosts, and the SWA composite's explicit refusal on gpt-oss. Admitting
|
||||
it again without a per-token move primitive re-breaks every family."""
|
||||
for backend in self.DSPARK_BACKENDS:
|
||||
self.assertFalse(
|
||||
_accepts("NGRAM", backend=backend),
|
||||
f"NGRAM must stay refused (backend={backend})",
|
||||
)
|
||||
|
||||
def test_spec_off_admitted(self):
|
||||
"""The gate constrains only speculative configurations; spec-off must
|
||||
keep booting regardless of family."""
|
||||
for is_hybrid_swa in (True, False):
|
||||
self.assertTrue(_accepts(None, is_hybrid_swa=is_hybrid_swa))
|
||||
|
||||
def test_unaudited_algorithms_refused(self):
|
||||
"""Every other algorithm stays out until its verify id rails are
|
||||
audited -- on every family."""
|
||||
for algorithm in self.UNAUDITED_ALGORITHMS:
|
||||
def test_unaudited_algorithms_and_tree_drafting_refused(self):
|
||||
"""NGRAM and tree drafting relocate accepted tokens one at a time
|
||||
inside the target pool, which the unified pool's page-granular move
|
||||
cannot express."""
|
||||
for algorithm in ("NGRAM", "STANDALONE"):
|
||||
for is_hybrid_swa in (True, False):
|
||||
self.assertFalse(_accepts(algorithm, is_hybrid_swa=is_hybrid_swa))
|
||||
for algorithm in ("DSPARK", "EAGLE"):
|
||||
for topk in (2, 4):
|
||||
self.assertFalse(_accepts(algorithm, topk=topk), f"{algorithm} {topk}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -281,6 +281,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase):
|
||||
hidden_states=object(), mm_input_embeds=None
|
||||
),
|
||||
next_token_ids=object(),
|
||||
kv_loc_plan=None,
|
||||
)
|
||||
verify_input = object()
|
||||
|
||||
@@ -306,7 +307,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase):
|
||||
self.assertEqual(kwargs["grammar_barrier"], "grammar")
|
||||
return result
|
||||
|
||||
def extend(current, *args):
|
||||
def extend(current, *args, **kwargs):
|
||||
record("draft_extend", current)
|
||||
return object()
|
||||
|
||||
|
||||
@@ -93,9 +93,7 @@ class DSparkTargetHiddenProjectionTest(CustomTestCase):
|
||||
draft_model=draft_model,
|
||||
draft_model_runner=SimpleNamespace(
|
||||
token_to_kv_pool=pool,
|
||||
kv_index_translator=SimpleNamespace(
|
||||
translate_full_attn_ids=lambda ids: ids
|
||||
),
|
||||
kv_index_translator=SimpleNamespace(),
|
||||
),
|
||||
model_runner=SimpleNamespace(device=torch.device("cpu")),
|
||||
device=torch.device("cpu"),
|
||||
|
||||
@@ -229,7 +229,9 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
|
||||
max_context_len=1,
|
||||
)
|
||||
worker.target_worker = SimpleNamespace(
|
||||
model_runner=SimpleNamespace(attn_backend=attn_backend)
|
||||
model_runner=SimpleNamespace(
|
||||
attn_backend=attn_backend, kv_index_translator=None
|
||||
)
|
||||
)
|
||||
draft_input = SimpleNamespace(
|
||||
bonus_tokens=torch.zeros((1,), dtype=torch.long, device=DEVICE),
|
||||
@@ -253,7 +255,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.speculative.eagle_worker_v2.prepare_for_draft",
|
||||
return_value=(forward_batch, True),
|
||||
return_value=(forward_batch, True, None),
|
||||
),
|
||||
):
|
||||
worker.draft(batch)
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A hand-built ForwardBatch view must carry the write rail.
|
||||
|
||||
BUG REGRESSION. The speculative draft-extend cuda-graph runners do not hand a
|
||||
real ForwardBatch to `init_forward_metadata_out_graph`; they build a
|
||||
`SimpleNamespace` view over their capture-stable buffers. On a translating
|
||||
pool that hook reaches `fill_capture_write_loc`, which reads
|
||||
`out_cache_loc_virtual` -- the pre-translate write loc -- and translates it
|
||||
into the backend's capture buffer.
|
||||
|
||||
A view that omits the field fails two ways, and the quiet one is worse. A
|
||||
`SimpleNamespace` raises AttributeError, which at least crashes the cell. But
|
||||
`fill_capture_write_loc` also accepts the field being None, and answers it by
|
||||
zeroing the buffer -- page-0 sink ids. So a view that "fixes" the crash by
|
||||
passing None sends every draft-extend KV write to the sink, and speculative
|
||||
decoding rejects the resulting drafts instead of failing: it surfaces as
|
||||
accept length decaying toward 1.0, not as an error.
|
||||
|
||||
Three runners had the omission independently, so this guards the shape rather
|
||||
than the three sites: any `SimpleNamespace` view that reaches
|
||||
`init_forward_metadata_out_graph` must name `out_cache_loc_virtual`.
|
||||
|
||||
The field is only read on a unified (translating) pool, which is why the
|
||||
default builds never caught it.
|
||||
|
||||
CPU-only, no torch.
|
||||
|
||||
python -m pytest test/registered/unit/spec/test_hand_built_view_write_rail.py -v
|
||||
"""
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
_SPEC_DIR = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python"
|
||||
/ "sglang"
|
||||
/ "srt"
|
||||
/ "speculative"
|
||||
)
|
||||
_HOOK = "init_forward_metadata_out_graph"
|
||||
_RAIL = "out_cache_loc_virtual"
|
||||
|
||||
|
||||
def _views_missing_the_rail(path: pathlib.Path):
|
||||
"""(function, variable, lineno) per hand-built view that drops the rail.
|
||||
|
||||
A view counts when it is a `SimpleNamespace(...)` bound to a name that is
|
||||
later passed to the hook in the same function -- that is exactly the shape
|
||||
the runners use, and it keeps the walk from flagging namespaces built for
|
||||
unrelated purposes.
|
||||
"""
|
||||
tree = ast.parse(path.read_text())
|
||||
offenders = []
|
||||
for fn in ast.walk(tree):
|
||||
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
# Names handed to the hook anywhere in this function.
|
||||
handed = set()
|
||||
for node in ast.walk(fn):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == _HOOK
|
||||
):
|
||||
for arg in node.args:
|
||||
if isinstance(arg, ast.Name):
|
||||
handed.add(arg.id)
|
||||
if not handed:
|
||||
continue
|
||||
for node in ast.walk(fn):
|
||||
if not (isinstance(node, ast.Assign) and isinstance(node.value, ast.Call)):
|
||||
continue
|
||||
func = node.value.func
|
||||
name = (
|
||||
func.attr
|
||||
if isinstance(func, ast.Attribute)
|
||||
else getattr(func, "id", None)
|
||||
)
|
||||
if name != "SimpleNamespace":
|
||||
continue
|
||||
for target in node.targets:
|
||||
if not (isinstance(target, ast.Name) and target.id in handed):
|
||||
continue
|
||||
# A literal None is the quiet failure: the hook zeroes the
|
||||
# capture buffer, routing every write to the page-0 sink.
|
||||
if not any(
|
||||
kw.arg == _RAIL
|
||||
and not (
|
||||
isinstance(kw.value, ast.Constant) and kw.value.value is None
|
||||
)
|
||||
for kw in node.value.keywords
|
||||
):
|
||||
offenders.append((fn.name, target.id, node.lineno))
|
||||
return sorted(offenders, key=lambda o: o[2])
|
||||
|
||||
|
||||
class TestHandBuiltViewWriteRail(unittest.TestCase):
|
||||
def test_every_hand_built_view_carries_the_rail(self):
|
||||
offenders = []
|
||||
for path in sorted(_SPEC_DIR.rglob("*.py")):
|
||||
for fn, var, lineno in _views_missing_the_rail(path):
|
||||
offenders.append(f"{path.name}:{lineno} {fn}() -> {var}")
|
||||
self.assertEqual(
|
||||
offenders,
|
||||
[],
|
||||
f"hand-built ForwardBatch view reaches {_HOOK} without {_RAIL}; on a "
|
||||
"unified pool the draft-extend writes land on the page-0 sink: "
|
||||
+ "; ".join(offenders),
|
||||
)
|
||||
|
||||
def test_detector_catches_a_view_that_drops_the_rail(self):
|
||||
"""The detector must catch the shape it guards, else it passes green
|
||||
forever after a refactor of the walk above."""
|
||||
src = (
|
||||
"class R:\n"
|
||||
" def replay(self, fb):\n"
|
||||
" v = SimpleNamespace(\n"
|
||||
" batch_size=8, out_cache_loc=self.buffers.out_cache_loc\n"
|
||||
" )\n"
|
||||
" self.backend.init_forward_metadata_out_graph(v)\n"
|
||||
)
|
||||
self.assertEqual(
|
||||
_views_missing_the_rail(self._probe(src)), [("replay", "v", 3)]
|
||||
)
|
||||
|
||||
def test_detector_catches_a_view_that_passes_none(self):
|
||||
"""None is accepted by `fill_capture_write_loc` and becomes the page-0
|
||||
sink, so naming the rail with None is the omission in disguise."""
|
||||
src = (
|
||||
"class R:\n"
|
||||
" def replay(self, fb):\n"
|
||||
" v = SimpleNamespace(batch_size=8, out_cache_loc_virtual=None)\n"
|
||||
" self.backend.init_forward_metadata_out_graph(v)\n"
|
||||
)
|
||||
self.assertEqual(
|
||||
_views_missing_the_rail(self._probe(src)), [("replay", "v", 3)]
|
||||
)
|
||||
|
||||
def test_detector_accepts_a_view_that_carries_it(self):
|
||||
"""...and must NOT fire once the rail is named, else it is unfixable."""
|
||||
src = (
|
||||
"class R:\n"
|
||||
" def replay(self, fb):\n"
|
||||
" v = SimpleNamespace(\n"
|
||||
" batch_size=8,\n"
|
||||
" out_cache_loc=self.buffers.out_cache_loc,\n"
|
||||
" out_cache_loc_virtual=fb.out_cache_loc_virtual,\n"
|
||||
" )\n"
|
||||
" self.backend.init_forward_metadata_out_graph(v)\n"
|
||||
)
|
||||
self.assertEqual(_views_missing_the_rail(self._probe(src)), [])
|
||||
|
||||
def test_detector_ignores_a_namespace_that_never_reaches_the_hook(self):
|
||||
"""A SimpleNamespace built for something else is not a view."""
|
||||
src = (
|
||||
"class R:\n"
|
||||
" def replay(self, fb):\n"
|
||||
" cfg = SimpleNamespace(batch_size=8)\n"
|
||||
" return cfg\n"
|
||||
)
|
||||
self.assertEqual(_views_missing_the_rail(self._probe(src)), [])
|
||||
|
||||
def _probe(self, src):
|
||||
# A temp dir, not the CWD: the checkout may be read-only.
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
scratch = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, scratch, ignore_errors=True)
|
||||
tmp = pathlib.Path(scratch) / (self.id().replace(".", "_") + ".py")
|
||||
tmp.write_text(src)
|
||||
return tmp
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user