[unified-memory] Name the fused KV translate for what it computes (7/7) (#40331)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:14:03 -07:00
committed by GitHub
co-authored by Cheng Wan
parent df530479c6
commit 2a3655fea1
28 changed files with 119 additions and 121 deletions
@@ -39,7 +39,7 @@ def create_flashinfer_kv_indices_triton(
``ENTRY_PAGE_SIZE == 1`` (default): the source table is token-granular and
entries are emitted verbatim -- byte-identical to the historical kernel.
``ENTRY_PAGE_SIZE == ps``: the source is the translator's PAGE-granular
read table (entries already kernel-facing page ids); token ids are rebuilt
read table (entries already physical page ids); token ids are rebuilt
as ``token = entry * ps + pos % ps``, exact because converting an id keeps
its offset inside the page.
``TOKEN_BLOCK_PARALLEL`` (default False): launched on a 2D grid
+1 -1
View File
@@ -18,7 +18,7 @@ _TRITON_KERNELS = [
("virtual_slot", "alloc_bind_inplace"),
("virtual_slot", "free_unbind_inplace"),
("virtual_slot", "bind_inplace"),
("virtual_slot", "write_loc_to_kernel_ids"),
("virtual_slot", "translate_token_ids"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
@@ -193,14 +193,14 @@ def bind_inplace(
bind_inplace_kernel[grid](v, p, v2p, p2v, N, BLOCK=ALLOC_BIND_BLOCK)
WRITE_LOC_BLOCK = 512
TRANSLATE_TOKEN_IDS_BLOCK = 512
@triton.jit
def write_loc_to_kernel_id_kernel(
loc_ptr, # in: [N] int64 — WIDENED virtual token ids
def translate_token_ids_kernel(
loc_ptr, # in: [N] or strided (rows, cols), int32/int64 — virtual token ids, widened under DCP
v2p_ptr, # in: [num_pages + 1] int64 — virtual->physical page table
out_ptr, # out: [N] int64 — physical token ids
out_ptr, # out: shaped like loc, or packed [W] when W > N; int64 — physical token ids
N, # runtime: live element count
W, # runtime: lanes to write; [N, W) get 0
num_v_pages, # runtime: rows of v2p; a page at or past it is unmapped
@@ -264,13 +264,13 @@ def write_loc_to_kernel_id_kernel(
def _row_col_strides(t: torch.Tensor) -> Tuple[int, int]:
"""Element strides of `t` seen as ``(rows, cols)``; 0-d and 1-D are one row."""
"""Element strides of a 1-D or 2-D `t` seen as ``(rows, cols)``; 1-D is one row."""
if t.dim() == 2:
return t.stride(0), t.stride(1)
return 0, (t.stride(0) if t.dim() == 1 else 1)
return 0, t.stride(0)
def write_loc_to_kernel_ids(
def translate_token_ids(
*,
loc: torch.Tensor,
v2p: torch.Tensor,
@@ -280,7 +280,7 @@ def write_loc_to_kernel_ids(
out: Optional[torch.Tensor] = None,
out_width: Optional[int] = None,
) -> torch.Tensor:
"""One launch for the whole write-loc conversion; see the kernel.
"""Virtual token ids -> physical token ids in one launch; see the kernel.
``out`` is written in place when given (a captured graph records the
gather against a fixed ``data_ptr``), else a fresh int64 tensor is
@@ -297,26 +297,25 @@ def write_loc_to_kernel_ids(
out = torch.empty(loc.shape, dtype=torch.int64, device=loc.device)
width = N if out_width is None else int(out_width)
assert out.dtype == torch.int64, (
f"write_loc_to_kernel_ids: out dtype must be int64 (matches v2p), "
f"got {out.dtype}"
f"translate_token_ids: out dtype must be int64 (matches v2p), got {out.dtype}"
)
if out_width is None:
# `out` mirrors `loc` whatever its shape; a 2-D page table is legal.
assert out.shape == loc.shape, (
f"write_loc_to_kernel_ids: out shape {tuple(out.shape)} must match "
f"translate_token_ids: out shape {tuple(out.shape)} must match "
f"loc shape {tuple(loc.shape)}"
)
else:
assert loc.is_contiguous(), (
f"write_loc_to_kernel_ids: out_width writes a packed lane range, "
f"translate_token_ids: out_width writes a packed lane range, "
f"so loc must be packed; got stride {tuple(loc.stride())}"
)
assert out.dim() == 1 and out.is_contiguous() and out.numel() >= width, (
f"write_loc_to_kernel_ids: out_width needs a packed 1-D out of at "
f"translate_token_ids: out_width needs a packed 1-D out of at "
f"least {width}, got {tuple(out.shape)}"
)
assert width >= N, (
f"write_loc_to_kernel_ids: out_width {width} is under the batch's "
f"translate_token_ids: out_width {width} is under the batch's "
f"{N} locs, which would drop live rows"
)
if width == 0:
@@ -348,14 +347,14 @@ def write_loc_to_kernel_ids(
out_row_stride = out_col_stride = 1
else:
assert loc.dim() <= 2 and out.dim() <= 2, (
f"write_loc_to_kernel_ids: a strided loc/out is walked as 2-D, got "
f"translate_token_ids: a strided loc/out is walked as 2-D, got "
f"{loc.dim()}-D loc and {out.dim()}-D out; pass a contiguous view"
)
cols = loc.shape[-1] if loc.dim() == 2 else N
loc_row_stride, loc_col_stride = _row_col_strides(loc)
out_row_stride, out_col_stride = _row_col_strides(out)
grid = (triton.cdiv(width, WRITE_LOC_BLOCK),)
write_loc_to_kernel_id_kernel[grid](
grid = (triton.cdiv(width, TRANSLATE_TOKEN_IDS_BLOCK),)
translate_token_ids_kernel[grid](
loc,
v2p,
out,
@@ -371,6 +370,6 @@ def write_loc_to_kernel_ids(
DCP_SIZE=dcp_size,
DCP_RANK=dcp_rank,
STRIDED=not flat,
BLOCK=WRITE_LOC_BLOCK,
BLOCK=TRANSLATE_TOKEN_IDS_BLOCK,
)
return out
@@ -545,7 +545,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
assert cfg.speculative_algorithm in (None, "DSPARK"), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK (chain draft); other speculative algorithms are not yet "
"audited for the unified pool's virtual/kernel-facing loc translation. Got "
"audited for the unified pool's virtual-to-physical loc translation. Got "
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
)
if cfg.speculative_algorithm == "DSPARK":
@@ -565,7 +565,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
f"attention backends {sorted(spec_allowed)} for both prefill "
f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do "
"not translate speculative verify indices to the unified "
"pool's kernel-facing space yet."
"pool's physical ids yet."
)
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
@@ -383,7 +383,7 @@ class FlashAttentionBackend(AttentionBackend):
if not model_runner.model_config.is_local_attention_model:
return None
# Local attention re-translates metadata.page_table through the static
# full->swa map, which is meaningless on the unified pool's kernel tables.
# full->swa map, which is meaningless on the unified pool's physical tables.
assert not self.kv_index_translator.is_translating, (
"--enable-unified-memory does not support local-attention models "
"on the fa3/fa4 backend."
@@ -749,7 +749,7 @@ class FlashAttentionBackend(AttentionBackend):
# straight into these capture-stable buffers from the LIVE v2p, so
# a page relocated by compaction since capture is picked up. Same
# substitution the eager extend branch makes in its `_unified_read`
# fixup; `build_index_table` emits page-granular kernel-facing ids
# fixup; `build_index_table` emits page-granular physical ids
# directly, so there is no `// page_size` to undo.
self.kv_index_translator.build_index_table(
req_pool_indices=forward_batch.req_pool_indices[:bs],
@@ -1208,7 +1208,7 @@ class FlashAttentionBackend(AttentionBackend):
if swa_out_cache_loc is not None:
# The swa write loc was computed from the still-VIRTUAL
# loc at ForwardBatch construction; re-running the
# full->swa map on the kernel-facing loc would be garbage.
# full->swa map on the physical loc would be garbage.
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
@@ -1722,7 +1722,7 @@ class FlashInferIndicesUpdaterDecode:
req_pool_indices: torch.Tensor,
):
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
# entries are already swa-side kernel-facing ids, so the in-place
# entries are already swa-side physical ids, so the in-place
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
@@ -2155,7 +2155,7 @@ class FlashInferIndicesUpdaterPrefill:
):
bs = len(seq_lens)
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
# entries are already swa-side kernel-facing ids, so the in-place
# entries are already swa-side physical ids, so the in-place
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
@@ -480,7 +480,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
self.decode_cuda_graph_kv_indices = torch.full(
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
)
# Unified pool: capture-stable buffer for the kernel-facing KV write loc, filled
# Unified pool: capture-stable buffer for the physical KV write loc, filled
# out-of-graph in init_forward_metadata_out_graph so the in-graph
# set_mla_kv_buffer captures no translate.
if self.kv_index_translator.is_translating:
@@ -1151,7 +1151,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
q: torch.Tensor,
q_rope: torch.Tensor,
) -> Optional[torch.Tensor]:
"""Decode: scatter the KV row at ``loc`` (already kernel-facing) and
"""Decode: scatter the KV row at ``loc`` (already physical) and
build the [q_nope | q_rope] fmha query in one kernel launch (saves one
launch per MLA layer and keeps the PDL chain intact).
@@ -1340,7 +1340,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
layer, self._kv_write_loc(forward_batch), k, k_rope
)
else:
# eager (or static pool): out_cache_loc is kernel-facing.
# eager (or static pool): out_cache_loc is physical.
if (
merge_query
and self._fused_set_kv_concat_q
@@ -152,7 +152,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
full_allocator=self.full_attn_allocator,
swa_allocator=self.swa_attn_allocator,
)
# Size host pools in tokens; sub-pool `size` counts kernel-facing rows.
# Size host pools in tokens; sub-pool `size` counts physical rows.
kvcache.full_kv_pool.host_capacity_tokens = self._size_full
kvcache.swa_kv_pool.host_capacity_tokens = self._size_swa
for name, pool in (
@@ -163,7 +163,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
pool.host_capacity_tokens * unified_buffer.spec(name).entry_bytes()
)
# Full-attention transfers use virtual IDs. SWA transfers already use
# kernel-facing IDs from translate_loc_from_full_to_swa.
# physical IDs from translate_loc_from_full_to_swa.
kvcache.full_kv_pool.host_transfer_translate = (
self.full_attn_allocator.translate_kv_loc
)
@@ -306,7 +306,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
"""Virtual TOKEN ids -> full-sub-pool PHYSICAL token ids for the PD
transfer engine.
PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see
The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see
`UnifiedMHATokenToKVPool.get_contiguous_buf_infos`). Without this
override the base identity would put VIRTUAL ids on the wire, which
address real bytes and so corrupt silently rather than fail.
@@ -341,7 +341,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
"""Bind SWA pages to resident or newly loaded full-attention virtual IDs.
Bind before translating: unbound pages translate to the padding sink.
Return kernel-facing IDs, or None if capacity cannot be reclaimed.
Return physical IDs, or None if capacity cannot be reclaimed.
"""
ids = full_token_ids.to(torch.int64)
if ids.numel() == 0:
@@ -372,16 +372,16 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
lazy_compaction=self.lazy_compaction,
)
def translate_write_loc_for_kernel(
def translate_write_loc(
self,
loc: torch.Tensor,
*,
out: Optional[torch.Tensor] = None,
out_width: Optional[int] = None,
) -> torch.Tensor:
"""Widened virtual WRITE loc -> kernel-facing id. DCP is rejected for this
"""Widened virtual WRITE loc -> physical id. DCP is rejected for this
composite at argument validation, so it coincides with the read translate."""
return self.full_attn_allocator.translate_write_loc_for_kernel(
return self.full_attn_allocator.translate_write_loc(
loc, out=out, out_width=out_width
)
@@ -127,7 +127,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
self.mamba_allocator.available_size(),
)
# HiCache indexes the full sub-pool's per-layer views with kernel-facing IDs.
# HiCache indexes the full sub-pool's per-layer views with physical IDs.
kvcache.full_kv_pool.host_transfer_translate = (
self.full_attn_allocator.translate_kv_loc
)
@@ -273,15 +273,15 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
"""Page-level physical->virtual table of the full sub-pool."""
return self.full_attn_allocator.physical_to_virtual
def translate_write_loc_for_kernel(
def translate_write_loc(
self,
loc: torch.Tensor,
*,
out: Optional[torch.Tensor] = None,
out_width: Optional[int] = None,
) -> torch.Tensor:
"""Widened virtual WRITE loc -> DENSE id; see the sub-allocator's copy."""
return self.full_attn_allocator.translate_write_loc_for_kernel(
"""Widened virtual WRITE loc -> physical id; see the sub-allocator's copy."""
return self.full_attn_allocator.translate_write_loc(
loc, out=out, out_width=out_width
)
@@ -289,7 +289,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
self, kv_indices: torch.Tensor
) -> torch.Tensor:
"""Virtual TOKEN ids -> PHYSICAL token ids for the PD transfer engine.
PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see
The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see
`UnifiedMLATokenToKVPool.get_contiguous_buf_infos`)."""
# Defensive: `_validate_unified_memory_dcp` rejects this pairing at
# argument validation, so reaching it means a config path got past that.
@@ -44,7 +44,7 @@ from sglang.kernels.ops.memory.virtual_slot import (
alloc_bind_inplace,
bind_inplace,
free_unbind_inplace,
write_loc_to_kernel_ids,
translate_token_ids,
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -993,10 +993,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
*,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Virtual token ids -> physical token ids, which under the token-major
views ARE the kernel-facing ids:
"""Virtual token ids -> physical token ids, the ids kernels index the
per-layer views with:
kernel_id(t) = v2p[t // ps] * ps + t % ps
physical(t) = v2p[t // ps] * ps + t % ps
Under DCP the input is the DCP-collapsed id (`widened // dcp_size`, what
`KVIndexTranslator.translate_dcp_read_ids` hands down), so this works on
@@ -1020,9 +1020,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
out_width: Optional[int] = None,
) -> torch.Tensor:
"""One launch for the read and write conversions alike; see
`write_loc_to_kernel_ids`, which owns the `out=` dtype and shape
`translate_token_ids`, which owns the `out=` dtype and shape
contract for both."""
return write_loc_to_kernel_ids(
return translate_token_ids(
loc=loc,
v2p=self.virtual_to_physical,
page_size=self.pool_page_size,
@@ -1032,22 +1032,22 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
out_width=out_width,
)
def translate_write_loc_for_kernel(
def translate_write_loc(
self,
widened_loc: torch.Tensor,
*,
out: Optional[torch.Tensor] = None,
out_width: Optional[int] = None,
) -> torch.Tensor:
"""Widened virtual WRITE loc (`out_cache_loc`) -> kernel-facing id.
"""Widened virtual WRITE loc (`out_cache_loc`) -> physical id.
Reads arrive already DCP-collapsed, but `out_cache_loc` does not: it still
carries the owner rule in `loc % dcp_size`. Ids this rank does not own go
to kernel id 0, the padding sink every write kernel skips.
to id 0, the padding sink every write kernel skips.
"""
parallel = get_parallel()
dcp_size = parallel.attn_dcp_size if self.shards_under_dcp else 1
with record_function("MultiEndedAlloc.translate_write_loc_for_kernel"):
with record_function("MultiEndedAlloc.translate_write_loc"):
return self._translate_loc_fused(
widened_loc,
dcp_size=dcp_size,
@@ -1535,7 +1535,7 @@ class HybridCacheController(BaseHiCacheController):
continue
if entry.device_indices_from_anchor_fn is not None:
# Allocate independent pools first: their allocation/eviction
# can compact SWA before its kernel-facing IDs are captured.
# can compact SWA before its physical IDs are captured.
anchor_transfers.append((pool, entry))
continue
# device_alloc_fn / device_free_fn override entry.device_pool's
@@ -42,7 +42,7 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
token_id = entry * entry_page_size + pos % entry_page_size
WRITES, IN TWO PHASES. The full-side write loc is rebound to kernel-facing
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
ids at ForwardBatch construction - the earliest consumer can snapshot it
right after. The sliding-window write loc is derived at the same moment as
read table, into the same index table.
@@ -92,7 +92,7 @@ class KVIndexTable(msgspec.Struct, frozen=True):
row_ids: torch.Tensor # which row belongs to batch lane b
row_stride: int # stride between rows of `ids`, in elements
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
is_translated: bool # entries are already kernel-facing ids
is_translated: bool # entries are already physical ids
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
@@ -129,7 +129,7 @@ class KVIndexTranslator:
# are collapsed by the DCP index kernels, `out_cache_loc` still
# carries the owner rule in `loc % dcp_size`. Identity with the read
# translate when dcp_size == 1.
self._translate_write_full = alloc.translate_write_loc_for_kernel
self._translate_write_full = alloc.translate_write_loc
# DCP read ids stay WIDENED to the consumer: selecting this rank's
# share changes the length, so only the production site can do it.
self.defer_read_translate = get_parallel().attn_dcp_size > 1
@@ -179,7 +179,7 @@ class KVIndexTranslator:
@property
def reads_are_translated(self) -> bool:
"""Whether a read this translator fills comes out kernel-facing. False
"""Whether a read this translator fills comes out physical. False
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
``translate_dcp_read_ids`` to finish."""
return self.is_translating and not self.defer_read_translate
@@ -506,7 +506,7 @@ class KVIndexTranslator:
return self.is_translating or get_parallel().attn_dcp_size > 1
def translate_dcp_read_ids(self, widened_ids: torch.Tensor) -> torch.Tensor:
"""Widened logical READ ids -> kernel-facing ids, for either pool.
"""Widened logical READ ids -> physical ids, for either pool.
The one hook every DCP read-index production site calls; on a static
pool `widened // dcp_size` IS the whole virtual->physical translation.
@@ -519,7 +519,7 @@ class KVIndexTranslator:
def translate_full_attn_ids(
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""Virtual token ids -> kernel-facing full-attention ids (the identity
"""Virtual token ids -> physical full-attention ids (the identity
when no translation is needed, so callers never branch)."""
if not self.is_translating:
assert out is None, "passthrough translate takes no out="
+3 -3
View File
@@ -1747,8 +1747,8 @@ class KVWriteLoc:
All location info lives here (in the attention metadata), NOT in the pool:
- ``loc``: the generic per-token write location (``out_cache_loc``).
KERNEL-FACING on every pool: physical by allocation on non-unified
pools, rebound at ForwardBatch construction (``rebind_write_loc``) on
PHYSICAL on every pool: by allocation on non-unified pools,
rebound at ForwardBatch construction (``rebind_write_loc``) on
the unified pool.
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
otherwise); under the unified pool the translator derives it from the
@@ -4349,7 +4349,7 @@ class HybridLinearKVPool(KVCache):
dst_dtype: Optional[torch.dtype] = None,
):
assert self.use_mla, "get_mla_kv_buffer called when use_mla is False"
# Read door -- same kernel-facing contract as the write door: `loc` is
# Read door -- same physical-id contract as the write door: `loc` is
# a read-index tensor already translated at its production site
# (fetch_mha_one_shot_kv_indices / prepare_chunked_kv_indices); the
# pool never translates.
@@ -296,11 +296,11 @@ def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) -
return use_mla_backend or not model_config.has_asymmetric_kv
def _assert_kernel_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
"""Check if kernel-facing ids can flow through int32 read-index buffers."""
def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
"""Physical ids must fit the int32 read-index buffers."""
assert n_rows < 2**31, (
f"sub-pool {sub_pool_name!r}: kernel-facing id space has {n_rows} rows, "
f"exceeding the int32 bound (2^31) that read-index buffers assume. "
f"sub-pool {sub_pool_name!r}: {n_rows} physical ids exceed the int32 "
f"bound (2^31) that read-index buffers assume. "
"Reduce max_total_num_tokens."
)
@@ -521,7 +521,7 @@ class UnifiedKVPool:
page_size: int,
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
num_slots = max_slots // page_size * page_size
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
layout = spec.layout()
k_views, v_views = (
build_dense_views(
@@ -543,7 +543,7 @@ class UnifiedKVPool:
page_size: int,
) -> List[torch.Tensor]:
num_slots = max_slots // page_size * page_size
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
layout = spec.layout()
return build_dense_views(
self._raw,
@@ -1424,7 +1424,7 @@ def init_unified_mamba_pools(
# Only HybridLinearKVPool's retraction CPU-copy path uses this hook.
token_to_kv_pool._mamba_translate = mamba_slot_allocator.translate
# No full-KV translate hook is wired: both MLA doors now receive
# KERNEL-FACING ids -- writes from the ForwardBatch rebind, reads
# PHYSICAL ids -- writes from the ForwardBatch rebind, reads
# translated at their production sites.
logger.info(
@@ -1668,7 +1668,7 @@ class UnifiedSWAKVPool(SWAKVPool):
layer_id_override=pool_layer_id,
)
return
# Full layer: `loc` is already the full-side kernel-facing id, so an
# Full layer: `loc` is already the full-side physical id, so an
# explicit full_loc is a same-space alias -- only triton's captured path
# passes one (its capture-stable buffer).
if full_loc is None:
@@ -532,7 +532,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# The original sequence length without being chunked. Qwen-1M related.
orig_seq_lens: Optional[torch.Tensor] = None
# The write loc before `rebind_write_loc` replaced it with kernel-facing
# The write loc before `rebind_write_loc` replaced it with physical
# ids; a backend re-derives from it into its capture-stable buffer.
out_cache_loc_virtual: Optional[torch.Tensor] = None
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
+1 -1
View File
@@ -202,7 +202,7 @@ class BaseTopkCapturer:
forward_batch, can_run_graph, cuda_graph_batch
)
# get_topk reads req_to_token IDs; attention may have rebound the write
# loc to kernel-facing IDs, which are neither stable nor the same space.
# loc to physical IDs, which are neither stable nor the same space.
out_cache_loc = forward_batch.out_cache_loc_virtual
if out_cache_loc is None:
out_cache_loc = forward_batch.out_cache_loc