[unified-memory] Name the fused KV translate for what it computes (7/7) (#40331)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -39,7 +39,7 @@ def create_flashinfer_kv_indices_triton(
|
||||
``ENTRY_PAGE_SIZE == 1`` (default): the source table is token-granular and
|
||||
entries are emitted verbatim -- byte-identical to the historical kernel.
|
||||
``ENTRY_PAGE_SIZE == ps``: the source is the translator's PAGE-granular
|
||||
read table (entries already kernel-facing page ids); token ids are rebuilt
|
||||
read table (entries already physical page ids); token ids are rebuilt
|
||||
as ``token = entry * ps + pos % ps``, exact because converting an id keeps
|
||||
its offset inside the page.
|
||||
``TOKEN_BLOCK_PARALLEL`` (default False): launched on a 2D grid
|
||||
|
||||
@@ -18,7 +18,7 @@ _TRITON_KERNELS = [
|
||||
("virtual_slot", "alloc_bind_inplace"),
|
||||
("virtual_slot", "free_unbind_inplace"),
|
||||
("virtual_slot", "bind_inplace"),
|
||||
("virtual_slot", "write_loc_to_kernel_ids"),
|
||||
("virtual_slot", "translate_token_ids"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
|
||||
@@ -193,14 +193,14 @@ def bind_inplace(
|
||||
bind_inplace_kernel[grid](v, p, v2p, p2v, N, BLOCK=ALLOC_BIND_BLOCK)
|
||||
|
||||
|
||||
WRITE_LOC_BLOCK = 512
|
||||
TRANSLATE_TOKEN_IDS_BLOCK = 512
|
||||
|
||||
|
||||
@triton.jit
|
||||
def write_loc_to_kernel_id_kernel(
|
||||
loc_ptr, # in: [N] int64 — WIDENED virtual token ids
|
||||
def translate_token_ids_kernel(
|
||||
loc_ptr, # in: [N] or strided (rows, cols), int32/int64 — virtual token ids, widened under DCP
|
||||
v2p_ptr, # in: [num_pages + 1] int64 — virtual->physical page table
|
||||
out_ptr, # out: [N] int64 — physical token ids
|
||||
out_ptr, # out: shaped like loc, or packed [W] when W > N; int64 — physical token ids
|
||||
N, # runtime: live element count
|
||||
W, # runtime: lanes to write; [N, W) get 0
|
||||
num_v_pages, # runtime: rows of v2p; a page at or past it is unmapped
|
||||
@@ -264,13 +264,13 @@ def write_loc_to_kernel_id_kernel(
|
||||
|
||||
|
||||
def _row_col_strides(t: torch.Tensor) -> Tuple[int, int]:
|
||||
"""Element strides of `t` seen as ``(rows, cols)``; 0-d and 1-D are one row."""
|
||||
"""Element strides of a 1-D or 2-D `t` seen as ``(rows, cols)``; 1-D is one row."""
|
||||
if t.dim() == 2:
|
||||
return t.stride(0), t.stride(1)
|
||||
return 0, (t.stride(0) if t.dim() == 1 else 1)
|
||||
return 0, t.stride(0)
|
||||
|
||||
|
||||
def write_loc_to_kernel_ids(
|
||||
def translate_token_ids(
|
||||
*,
|
||||
loc: torch.Tensor,
|
||||
v2p: torch.Tensor,
|
||||
@@ -280,7 +280,7 @@ def write_loc_to_kernel_ids(
|
||||
out: Optional[torch.Tensor] = None,
|
||||
out_width: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""One launch for the whole write-loc conversion; see the kernel.
|
||||
"""Virtual token ids -> physical token ids in one launch; see the kernel.
|
||||
|
||||
``out`` is written in place when given (a captured graph records the
|
||||
gather against a fixed ``data_ptr``), else a fresh int64 tensor is
|
||||
@@ -297,26 +297,25 @@ def write_loc_to_kernel_ids(
|
||||
out = torch.empty(loc.shape, dtype=torch.int64, device=loc.device)
|
||||
width = N if out_width is None else int(out_width)
|
||||
assert out.dtype == torch.int64, (
|
||||
f"write_loc_to_kernel_ids: out dtype must be int64 (matches v2p), "
|
||||
f"got {out.dtype}"
|
||||
f"translate_token_ids: out dtype must be int64 (matches v2p), got {out.dtype}"
|
||||
)
|
||||
if out_width is None:
|
||||
# `out` mirrors `loc` whatever its shape; a 2-D page table is legal.
|
||||
assert out.shape == loc.shape, (
|
||||
f"write_loc_to_kernel_ids: out shape {tuple(out.shape)} must match "
|
||||
f"translate_token_ids: out shape {tuple(out.shape)} must match "
|
||||
f"loc shape {tuple(loc.shape)}"
|
||||
)
|
||||
else:
|
||||
assert loc.is_contiguous(), (
|
||||
f"write_loc_to_kernel_ids: out_width writes a packed lane range, "
|
||||
f"translate_token_ids: out_width writes a packed lane range, "
|
||||
f"so loc must be packed; got stride {tuple(loc.stride())}"
|
||||
)
|
||||
assert out.dim() == 1 and out.is_contiguous() and out.numel() >= width, (
|
||||
f"write_loc_to_kernel_ids: out_width needs a packed 1-D out of at "
|
||||
f"translate_token_ids: out_width needs a packed 1-D out of at "
|
||||
f"least {width}, got {tuple(out.shape)}"
|
||||
)
|
||||
assert width >= N, (
|
||||
f"write_loc_to_kernel_ids: out_width {width} is under the batch's "
|
||||
f"translate_token_ids: out_width {width} is under the batch's "
|
||||
f"{N} locs, which would drop live rows"
|
||||
)
|
||||
if width == 0:
|
||||
@@ -348,14 +347,14 @@ def write_loc_to_kernel_ids(
|
||||
out_row_stride = out_col_stride = 1
|
||||
else:
|
||||
assert loc.dim() <= 2 and out.dim() <= 2, (
|
||||
f"write_loc_to_kernel_ids: a strided loc/out is walked as 2-D, got "
|
||||
f"translate_token_ids: a strided loc/out is walked as 2-D, got "
|
||||
f"{loc.dim()}-D loc and {out.dim()}-D out; pass a contiguous view"
|
||||
)
|
||||
cols = loc.shape[-1] if loc.dim() == 2 else N
|
||||
loc_row_stride, loc_col_stride = _row_col_strides(loc)
|
||||
out_row_stride, out_col_stride = _row_col_strides(out)
|
||||
grid = (triton.cdiv(width, WRITE_LOC_BLOCK),)
|
||||
write_loc_to_kernel_id_kernel[grid](
|
||||
grid = (triton.cdiv(width, TRANSLATE_TOKEN_IDS_BLOCK),)
|
||||
translate_token_ids_kernel[grid](
|
||||
loc,
|
||||
v2p,
|
||||
out,
|
||||
@@ -371,6 +370,6 @@ def write_loc_to_kernel_ids(
|
||||
DCP_SIZE=dcp_size,
|
||||
DCP_RANK=dcp_rank,
|
||||
STRIDED=not flat,
|
||||
BLOCK=WRITE_LOC_BLOCK,
|
||||
BLOCK=TRANSLATE_TOKEN_IDS_BLOCK,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -545,7 +545,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
assert cfg.speculative_algorithm in (None, "DSPARK"), (
|
||||
"--enable-unified-memory only supports --speculative-algorithm "
|
||||
"DSPARK (chain draft); other speculative algorithms are not yet "
|
||||
"audited for the unified pool's virtual/kernel-facing loc translation. Got "
|
||||
"audited for the unified pool's virtual-to-physical loc translation. Got "
|
||||
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
|
||||
)
|
||||
if cfg.speculative_algorithm == "DSPARK":
|
||||
@@ -565,7 +565,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
f"attention backends {sorted(spec_allowed)} for both prefill "
|
||||
f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do "
|
||||
"not translate speculative verify indices to the unified "
|
||||
"pool's kernel-facing space yet."
|
||||
"pool's physical ids yet."
|
||||
)
|
||||
assert not cfg.enable_two_batch_overlap, (
|
||||
"--enable-unified-memory does not support --enable-two-batch-overlap: "
|
||||
|
||||
@@ -383,7 +383,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if not model_runner.model_config.is_local_attention_model:
|
||||
return None
|
||||
# Local attention re-translates metadata.page_table through the static
|
||||
# full->swa map, which is meaningless on the unified pool's kernel tables.
|
||||
# full->swa map, which is meaningless on the unified pool's physical tables.
|
||||
assert not self.kv_index_translator.is_translating, (
|
||||
"--enable-unified-memory does not support local-attention models "
|
||||
"on the fa3/fa4 backend."
|
||||
@@ -749,7 +749,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# straight into these capture-stable buffers from the LIVE v2p, so
|
||||
# a page relocated by compaction since capture is picked up. Same
|
||||
# substitution the eager extend branch makes in its `_unified_read`
|
||||
# fixup; `build_index_table` emits page-granular kernel-facing ids
|
||||
# fixup; `build_index_table` emits page-granular physical ids
|
||||
# directly, so there is no `// page_size` to undo.
|
||||
self.kv_index_translator.build_index_table(
|
||||
req_pool_indices=forward_batch.req_pool_indices[:bs],
|
||||
@@ -1208,7 +1208,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if swa_out_cache_loc is not None:
|
||||
# The swa write loc was computed from the still-VIRTUAL
|
||||
# loc at ForwardBatch construction; re-running the
|
||||
# full->swa map on the kernel-facing loc would be garbage.
|
||||
# full->swa map on the physical loc would be garbage.
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
|
||||
@@ -1722,7 +1722,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
# entries are already swa-side kernel-facing ids, so the in-place
|
||||
# entries are already swa-side physical ids, so the in-place
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
@@ -2155,7 +2155,7 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
):
|
||||
bs = len(seq_lens)
|
||||
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
|
||||
# entries are already swa-side kernel-facing ids, so the in-place
|
||||
# entries are already swa-side physical ids, so the in-place
|
||||
# full->swa translate below must not run on top of them.
|
||||
translator = self.attn_backend.kv_index_translator
|
||||
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
|
||||
|
||||
@@ -480,7 +480,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
self.decode_cuda_graph_kv_indices = torch.full(
|
||||
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
# Unified pool: capture-stable buffer for the kernel-facing KV write loc, filled
|
||||
# Unified pool: capture-stable buffer for the physical KV write loc, filled
|
||||
# out-of-graph in init_forward_metadata_out_graph so the in-graph
|
||||
# set_mla_kv_buffer captures no translate.
|
||||
if self.kv_index_translator.is_translating:
|
||||
@@ -1151,7 +1151,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
q: torch.Tensor,
|
||||
q_rope: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Decode: scatter the KV row at ``loc`` (already kernel-facing) and
|
||||
"""Decode: scatter the KV row at ``loc`` (already physical) and
|
||||
build the [q_nope | q_rope] fmha query in one kernel launch (saves one
|
||||
launch per MLA layer and keeps the PDL chain intact).
|
||||
|
||||
@@ -1340,7 +1340,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
layer, self._kv_write_loc(forward_batch), k, k_rope
|
||||
)
|
||||
else:
|
||||
# eager (or static pool): out_cache_loc is kernel-facing.
|
||||
# eager (or static pool): out_cache_loc is physical.
|
||||
if (
|
||||
merge_query
|
||||
and self._fused_set_kv_concat_q
|
||||
|
||||
@@ -152,7 +152,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
full_allocator=self.full_attn_allocator,
|
||||
swa_allocator=self.swa_attn_allocator,
|
||||
)
|
||||
# Size host pools in tokens; sub-pool `size` counts kernel-facing rows.
|
||||
# Size host pools in tokens; sub-pool `size` counts physical rows.
|
||||
kvcache.full_kv_pool.host_capacity_tokens = self._size_full
|
||||
kvcache.swa_kv_pool.host_capacity_tokens = self._size_swa
|
||||
for name, pool in (
|
||||
@@ -163,7 +163,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
pool.host_capacity_tokens * unified_buffer.spec(name).entry_bytes()
|
||||
)
|
||||
# Full-attention transfers use virtual IDs. SWA transfers already use
|
||||
# kernel-facing IDs from translate_loc_from_full_to_swa.
|
||||
# physical IDs from translate_loc_from_full_to_swa.
|
||||
kvcache.full_kv_pool.host_transfer_translate = (
|
||||
self.full_attn_allocator.translate_kv_loc
|
||||
)
|
||||
@@ -306,7 +306,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
"""Virtual TOKEN ids -> full-sub-pool PHYSICAL token ids for the PD
|
||||
transfer engine.
|
||||
|
||||
PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see
|
||||
The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see
|
||||
`UnifiedMHATokenToKVPool.get_contiguous_buf_infos`). Without this
|
||||
override the base identity would put VIRTUAL ids on the wire, which
|
||||
address real bytes and so corrupt silently rather than fail.
|
||||
@@ -341,7 +341,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
|
||||
|
||||
Bind before translating: unbound pages translate to the padding sink.
|
||||
Return kernel-facing IDs, or None if capacity cannot be reclaimed.
|
||||
Return physical IDs, or None if capacity cannot be reclaimed.
|
||||
"""
|
||||
ids = full_token_ids.to(torch.int64)
|
||||
if ids.numel() == 0:
|
||||
@@ -372,16 +372,16 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def translate_write_loc_for_kernel(
|
||||
def translate_write_loc(
|
||||
self,
|
||||
loc: torch.Tensor,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
out_width: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Widened virtual WRITE loc -> kernel-facing id. DCP is rejected for this
|
||||
"""Widened virtual WRITE loc -> physical id. DCP is rejected for this
|
||||
composite at argument validation, so it coincides with the read translate."""
|
||||
return self.full_attn_allocator.translate_write_loc_for_kernel(
|
||||
return self.full_attn_allocator.translate_write_loc(
|
||||
loc, out=out, out_width=out_width
|
||||
)
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.mamba_allocator.available_size(),
|
||||
)
|
||||
|
||||
# HiCache indexes the full sub-pool's per-layer views with kernel-facing IDs.
|
||||
# HiCache indexes the full sub-pool's per-layer views with physical IDs.
|
||||
kvcache.full_kv_pool.host_transfer_translate = (
|
||||
self.full_attn_allocator.translate_kv_loc
|
||||
)
|
||||
@@ -273,15 +273,15 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
"""Page-level physical->virtual table of the full sub-pool."""
|
||||
return self.full_attn_allocator.physical_to_virtual
|
||||
|
||||
def translate_write_loc_for_kernel(
|
||||
def translate_write_loc(
|
||||
self,
|
||||
loc: torch.Tensor,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
out_width: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Widened virtual WRITE loc -> DENSE id; see the sub-allocator's copy."""
|
||||
return self.full_attn_allocator.translate_write_loc_for_kernel(
|
||||
"""Widened virtual WRITE loc -> physical id; see the sub-allocator's copy."""
|
||||
return self.full_attn_allocator.translate_write_loc(
|
||||
loc, out=out, out_width=out_width
|
||||
)
|
||||
|
||||
@@ -289,7 +289,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
self, kv_indices: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Virtual TOKEN ids -> PHYSICAL token ids for the PD transfer engine.
|
||||
PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see
|
||||
The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see
|
||||
`UnifiedMLATokenToKVPool.get_contiguous_buf_infos`)."""
|
||||
# Defensive: `_validate_unified_memory_dcp` rejects this pairing at
|
||||
# argument validation, so reaching it means a config path got past that.
|
||||
|
||||
@@ -44,7 +44,7 @@ from sglang.kernels.ops.memory.virtual_slot import (
|
||||
alloc_bind_inplace,
|
||||
bind_inplace,
|
||||
free_unbind_inplace,
|
||||
write_loc_to_kernel_ids,
|
||||
translate_token_ids,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
@@ -993,10 +993,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Virtual token ids -> physical token ids, which under the token-major
|
||||
views ARE the kernel-facing ids:
|
||||
"""Virtual token ids -> physical token ids, the ids kernels index the
|
||||
per-layer views with:
|
||||
|
||||
kernel_id(t) = v2p[t // ps] * ps + t % ps
|
||||
physical(t) = v2p[t // ps] * ps + t % ps
|
||||
|
||||
Under DCP the input is the DCP-collapsed id (`widened // dcp_size`, what
|
||||
`KVIndexTranslator.translate_dcp_read_ids` hands down), so this works on
|
||||
@@ -1020,9 +1020,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
out_width: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""One launch for the read and write conversions alike; see
|
||||
`write_loc_to_kernel_ids`, which owns the `out=` dtype and shape
|
||||
`translate_token_ids`, which owns the `out=` dtype and shape
|
||||
contract for both."""
|
||||
return write_loc_to_kernel_ids(
|
||||
return translate_token_ids(
|
||||
loc=loc,
|
||||
v2p=self.virtual_to_physical,
|
||||
page_size=self.pool_page_size,
|
||||
@@ -1032,22 +1032,22 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
out_width=out_width,
|
||||
)
|
||||
|
||||
def translate_write_loc_for_kernel(
|
||||
def translate_write_loc(
|
||||
self,
|
||||
widened_loc: torch.Tensor,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
out_width: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Widened virtual WRITE loc (`out_cache_loc`) -> kernel-facing id.
|
||||
"""Widened virtual WRITE loc (`out_cache_loc`) -> physical id.
|
||||
|
||||
Reads arrive already DCP-collapsed, but `out_cache_loc` does not: it still
|
||||
carries the owner rule in `loc % dcp_size`. Ids this rank does not own go
|
||||
to kernel id 0, the padding sink every write kernel skips.
|
||||
to id 0, the padding sink every write kernel skips.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
dcp_size = parallel.attn_dcp_size if self.shards_under_dcp else 1
|
||||
with record_function("MultiEndedAlloc.translate_write_loc_for_kernel"):
|
||||
with record_function("MultiEndedAlloc.translate_write_loc"):
|
||||
return self._translate_loc_fused(
|
||||
widened_loc,
|
||||
dcp_size=dcp_size,
|
||||
|
||||
@@ -1535,7 +1535,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
continue
|
||||
if entry.device_indices_from_anchor_fn is not None:
|
||||
# Allocate independent pools first: their allocation/eviction
|
||||
# can compact SWA before its kernel-facing IDs are captured.
|
||||
# can compact SWA before its physical IDs are captured.
|
||||
anchor_transfers.append((pool, entry))
|
||||
continue
|
||||
# device_alloc_fn / device_free_fn override entry.device_pool's
|
||||
|
||||
@@ -42,7 +42,7 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
|
||||
|
||||
token_id = entry * entry_page_size + pos % entry_page_size
|
||||
|
||||
WRITES, IN TWO PHASES. The full-side write loc is rebound to kernel-facing
|
||||
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
|
||||
ids at ForwardBatch construction - the earliest consumer can snapshot it
|
||||
right after. The sliding-window write loc is derived at the same moment as
|
||||
read table, into the same index table.
|
||||
@@ -92,7 +92,7 @@ class KVIndexTable(msgspec.Struct, frozen=True):
|
||||
row_ids: torch.Tensor # which row belongs to batch lane b
|
||||
row_stride: int # stride between rows of `ids`, in elements
|
||||
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
|
||||
is_translated: bool # entries are already kernel-facing ids
|
||||
is_translated: bool # entries are already physical ids
|
||||
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
|
||||
|
||||
|
||||
@@ -129,7 +129,7 @@ class KVIndexTranslator:
|
||||
# are collapsed by the DCP index kernels, `out_cache_loc` still
|
||||
# carries the owner rule in `loc % dcp_size`. Identity with the read
|
||||
# translate when dcp_size == 1.
|
||||
self._translate_write_full = alloc.translate_write_loc_for_kernel
|
||||
self._translate_write_full = alloc.translate_write_loc
|
||||
# DCP read ids stay WIDENED to the consumer: selecting this rank's
|
||||
# share changes the length, so only the production site can do it.
|
||||
self.defer_read_translate = get_parallel().attn_dcp_size > 1
|
||||
@@ -179,7 +179,7 @@ class KVIndexTranslator:
|
||||
|
||||
@property
|
||||
def reads_are_translated(self) -> bool:
|
||||
"""Whether a read this translator fills comes out kernel-facing. False
|
||||
"""Whether a read this translator fills comes out physical. False
|
||||
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
|
||||
``translate_dcp_read_ids`` to finish."""
|
||||
return self.is_translating and not self.defer_read_translate
|
||||
@@ -506,7 +506,7 @@ class KVIndexTranslator:
|
||||
return self.is_translating or get_parallel().attn_dcp_size > 1
|
||||
|
||||
def translate_dcp_read_ids(self, widened_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Widened logical READ ids -> kernel-facing ids, for either pool.
|
||||
"""Widened logical READ ids -> physical ids, for either pool.
|
||||
|
||||
The one hook every DCP read-index production site calls; on a static
|
||||
pool `widened // dcp_size` IS the whole virtual->physical translation.
|
||||
@@ -519,7 +519,7 @@ class KVIndexTranslator:
|
||||
def translate_full_attn_ids(
|
||||
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""Virtual token ids -> kernel-facing full-attention ids (the identity
|
||||
"""Virtual token ids -> physical full-attention ids (the identity
|
||||
when no translation is needed, so callers never branch)."""
|
||||
if not self.is_translating:
|
||||
assert out is None, "passthrough translate takes no out="
|
||||
|
||||
@@ -1747,8 +1747,8 @@ class KVWriteLoc:
|
||||
|
||||
All location info lives here (in the attention metadata), NOT in the pool:
|
||||
- ``loc``: the generic per-token write location (``out_cache_loc``).
|
||||
KERNEL-FACING on every pool: physical by allocation on non-unified
|
||||
pools, rebound at ForwardBatch construction (``rebind_write_loc``) on
|
||||
PHYSICAL on every pool: by allocation on non-unified pools,
|
||||
rebound at ForwardBatch construction (``rebind_write_loc``) on
|
||||
the unified pool.
|
||||
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
|
||||
otherwise); under the unified pool the translator derives it from the
|
||||
@@ -4349,7 +4349,7 @@ class HybridLinearKVPool(KVCache):
|
||||
dst_dtype: Optional[torch.dtype] = None,
|
||||
):
|
||||
assert self.use_mla, "get_mla_kv_buffer called when use_mla is False"
|
||||
# Read door -- same kernel-facing contract as the write door: `loc` is
|
||||
# Read door -- same physical-id contract as the write door: `loc` is
|
||||
# a read-index tensor already translated at its production site
|
||||
# (fetch_mha_one_shot_kv_indices / prepare_chunked_kv_indices); the
|
||||
# pool never translates.
|
||||
|
||||
@@ -296,11 +296,11 @@ def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) -
|
||||
return use_mla_backend or not model_config.has_asymmetric_kv
|
||||
|
||||
|
||||
def _assert_kernel_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
|
||||
"""Check if kernel-facing ids can flow through int32 read-index buffers."""
|
||||
def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
|
||||
"""Physical ids must fit the int32 read-index buffers."""
|
||||
assert n_rows < 2**31, (
|
||||
f"sub-pool {sub_pool_name!r}: kernel-facing id space has {n_rows} rows, "
|
||||
f"exceeding the int32 bound (2^31) that read-index buffers assume. "
|
||||
f"sub-pool {sub_pool_name!r}: {n_rows} physical ids exceed the int32 "
|
||||
f"bound (2^31) that read-index buffers assume. "
|
||||
"Reduce max_total_num_tokens."
|
||||
)
|
||||
|
||||
@@ -521,7 +521,7 @@ class UnifiedKVPool:
|
||||
page_size: int,
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
num_slots = max_slots // page_size * page_size
|
||||
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
layout = spec.layout()
|
||||
k_views, v_views = (
|
||||
build_dense_views(
|
||||
@@ -543,7 +543,7 @@ class UnifiedKVPool:
|
||||
page_size: int,
|
||||
) -> List[torch.Tensor]:
|
||||
num_slots = max_slots // page_size * page_size
|
||||
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
layout = spec.layout()
|
||||
return build_dense_views(
|
||||
self._raw,
|
||||
@@ -1424,7 +1424,7 @@ def init_unified_mamba_pools(
|
||||
# Only HybridLinearKVPool's retraction CPU-copy path uses this hook.
|
||||
token_to_kv_pool._mamba_translate = mamba_slot_allocator.translate
|
||||
# No full-KV translate hook is wired: both MLA doors now receive
|
||||
# KERNEL-FACING ids -- writes from the ForwardBatch rebind, reads
|
||||
# PHYSICAL ids -- writes from the ForwardBatch rebind, reads
|
||||
# translated at their production sites.
|
||||
|
||||
logger.info(
|
||||
@@ -1668,7 +1668,7 @@ class UnifiedSWAKVPool(SWAKVPool):
|
||||
layer_id_override=pool_layer_id,
|
||||
)
|
||||
return
|
||||
# Full layer: `loc` is already the full-side kernel-facing id, so an
|
||||
# Full layer: `loc` is already the full-side physical id, so an
|
||||
# explicit full_loc is a same-space alias -- only triton's captured path
|
||||
# passes one (its capture-stable buffer).
|
||||
if full_loc is None:
|
||||
|
||||
@@ -532,7 +532,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# The original sequence length without being chunked. Qwen-1M related.
|
||||
orig_seq_lens: Optional[torch.Tensor] = None
|
||||
|
||||
# The write loc before `rebind_write_loc` replaced it with kernel-facing
|
||||
# The write loc before `rebind_write_loc` replaced it with physical
|
||||
# ids; a backend re-derives from it into its capture-stable buffer.
|
||||
out_cache_loc_virtual: Optional[torch.Tensor] = None
|
||||
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
|
||||
|
||||
@@ -202,7 +202,7 @@ class BaseTopkCapturer:
|
||||
forward_batch, can_run_graph, cuda_graph_batch
|
||||
)
|
||||
# get_topk reads req_to_token IDs; attention may have rebound the write
|
||||
# loc to kernel-facing IDs, which are neither stable nor the same space.
|
||||
# loc to physical IDs, which are neither stable nor the same space.
|
||||
out_cache_loc = forward_batch.out_cache_loc_virtual
|
||||
if out_cache_loc is None:
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
|
||||
Reference in New Issue
Block a user