diff --git a/python/sglang/kernels/ops/memory/virtual_slot.py b/python/sglang/kernels/ops/memory/virtual_slot.py index 06ccbf60b8..f1c744c751 100644 --- a/python/sglang/kernels/ops/memory/virtual_slot.py +++ b/python/sglang/kernels/ops/memory/virtual_slot.py @@ -15,7 +15,7 @@ from __future__ import annotations -from typing import Optional +from typing import Optional, Tuple import torch import triton @@ -203,9 +203,16 @@ def write_loc_to_kernel_id_kernel( out_ptr, # out: [N] int64 — physical token ids N, # runtime: live element count W, # runtime: lanes to write; [N, W) get 0 + num_v_pages, # runtime: rows of v2p; a page at or past it is unmapped + cols, # runtime: row width of the (rows, cols) view; STRIDED only + loc_row_stride, # runtime: element strides of `loc`; STRIDED only + loc_col_stride, + out_row_stride, # runtime: element strides of `out`; STRIDED only + out_col_stride, PAGE_SIZE: tl.constexpr, DCP_SIZE: tl.constexpr, DCP_RANK: tl.constexpr, + STRIDED: tl.constexpr, BLOCK: tl.constexpr, ): """``physical(t) = v2p[t // ps] * ps + t % ps``, clamped at 0. @@ -221,12 +228,25 @@ def write_loc_to_kernel_id_kernel( Writing ``W > N`` lanes fills ``[N, W)`` with 0, the padding sink, so a caller may hand in a capture-stable buffer wider than this batch and have the stale tail cleared in the same launch. + + ``STRIDED`` walks `loc` and `out` as a ``(rows, cols)`` view, so a column + slice of a wider page table addresses correctly; a contiguous pair takes + the flat path and the stride arguments go unread. """ pid = tl.program_id(0) offs = pid * BLOCK + tl.arange(0, BLOCK) in_range = offs < W mask = offs < N - loc = tl.load(loc_ptr + offs, mask=mask, other=0).to(tl.int64) + if STRIDED: + wide = offs.to(tl.int64) + row = wide // cols + col = wide - row * cols + loc_off = row * loc_row_stride + col * loc_col_stride + out_off = row * out_row_stride + col * out_col_stride + else: + loc_off = offs + out_off = offs + loc = tl.load(loc_ptr + loc_off, mask=mask, other=0).to(tl.int64) keep = mask & (loc >= 0) if DCP_SIZE > 1: @@ -235,10 +255,19 @@ def write_loc_to_kernel_id_kernel( page = loc // PAGE_SIZE if PAGE_SIZE > 1 else loc offset = loc % PAGE_SIZE if PAGE_SIZE > 1 else 0 - # `keep` already excludes negatives, so the gather index is in range. + # Torch's gather bounds-checks and `tl.load` does not, so an id past the + # table is excluded here rather than read out of bounds. + keep = keep & (page < num_v_pages) phys = tl.load(v2p_ptr + tl.where(keep, page, 0), mask=mask, other=0).to(tl.int64) ids = tl.maximum(phys * PAGE_SIZE + offset, 0) - tl.store(out_ptr + offs, tl.where(keep, ids, 0), mask=in_range) + tl.store(out_ptr + out_off, tl.where(keep, ids, 0), mask=in_range) + + +def _row_col_strides(t: torch.Tensor) -> Tuple[int, int]: + """Element strides of `t` seen as ``(rows, cols)``; 0-d and 1-D are one row.""" + if t.dim() == 2: + return t.stride(0), t.stride(1) + return 0, (t.stride(0) if t.dim() == 1 else 1) def write_loc_to_kernel_ids( @@ -260,16 +289,12 @@ def write_loc_to_kernel_ids( ``out_width`` writes that many lanes rather than ``loc.numel()``, zeroing the ones past the batch; pass the captured tier's width to clear a stale - tail here. + tail here. That mode addresses a packed lane range, so it takes a + contiguous ``loc``; everywhere else a 1-D or 2-D strided view is legal. """ N = int(loc.numel()) - # Flat-indexed as `ptr + offs`, so a strided view is mis-addressed. - assert loc.is_contiguous(), ( - f"write_loc_to_kernel_ids: loc must be contiguous, got shape " - f"{tuple(loc.shape)} stride {tuple(loc.stride())}" - ) if out is None: - out = torch.empty_like(loc, dtype=torch.int64) + out = torch.empty(loc.shape, dtype=torch.int64, device=loc.device) width = N if out_width is None else int(out_width) assert out.dtype == torch.int64, ( f"write_loc_to_kernel_ids: out dtype must be int64 (matches v2p), " @@ -277,11 +302,15 @@ def write_loc_to_kernel_ids( ) if out_width is None: # `out` mirrors `loc` whatever its shape; a 2-D page table is legal. - assert out.shape == loc.shape and out.is_contiguous(), ( + assert out.shape == loc.shape, ( f"write_loc_to_kernel_ids: out shape {tuple(out.shape)} must match " f"loc shape {tuple(loc.shape)}" ) else: + assert loc.is_contiguous(), ( + f"write_loc_to_kernel_ids: out_width writes a packed lane range, " + f"so loc must be packed; got stride {tuple(loc.stride())}" + ) assert out.dim() == 1 and out.is_contiguous() and out.numel() >= width, ( f"write_loc_to_kernel_ids: out_width needs a packed 1-D out of at " f"least {width}, got {tuple(out.shape)}" @@ -293,19 +322,38 @@ def write_loc_to_kernel_ids( if width == 0: return out if not loc.is_cuda: - # Pure-torch reference; the allocator's unit tests run on CPU. + # Pure-torch reference; taken by the CPU unit tests and by every + # non-CUDA accelerator. big = loc.to(torch.int64) keep = big >= 0 if dcp_size > 1: keep = keep & (big % dcp_size == dcp_rank) big = torch.div(big, dcp_size, rounding_mode="floor") - page = torch.where(keep, torch.div(big, page_size, rounding_mode="floor"), 0) + page = torch.div(big, page_size, rounding_mode="floor") + keep = keep & (page < v2p.numel()) + page = torch.where(keep, page, 0) offset = big % page_size if page_size > 1 else 0 ids = (v2p[page] * page_size + offset).clamp_(min=0) - out[:N].copy_(torch.where(keep, ids, torch.zeros_like(ids))) - if width > N: - out[N:width].zero_() + ids = torch.where(keep, ids, torch.zeros_like(ids)) + if out_width is None: + out.copy_(ids) + else: + out[:N].copy_(ids) + if width > N: + out[N:width].zero_() return out + flat = loc.is_contiguous() and out.is_contiguous() + if flat: + cols = loc_row_stride = loc_col_stride = 1 + out_row_stride = out_col_stride = 1 + else: + assert loc.dim() <= 2 and out.dim() <= 2, ( + f"write_loc_to_kernel_ids: a strided loc/out is walked as 2-D, got " + f"{loc.dim()}-D loc and {out.dim()}-D out; pass a contiguous view" + ) + cols = loc.shape[-1] if loc.dim() == 2 else N + loc_row_stride, loc_col_stride = _row_col_strides(loc) + out_row_stride, out_col_stride = _row_col_strides(out) grid = (triton.cdiv(width, WRITE_LOC_BLOCK),) write_loc_to_kernel_id_kernel[grid]( loc, @@ -313,9 +361,16 @@ def write_loc_to_kernel_ids( out, N, width, + int(v2p.numel()), + cols, + loc_row_stride, + loc_col_stride, + out_row_stride, + out_col_stride, PAGE_SIZE=page_size, DCP_SIZE=dcp_size, DCP_RANK=dcp_rank, + STRIDED=not flat, BLOCK=WRITE_LOC_BLOCK, ) return out diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index cedfe8ba72..8b3fba1334 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -788,10 +788,11 @@ class FlashAttentionBackend(AttentionBackend): "full-CG prefill SWA write-location buffer", ) # Under the unified pool `out_cache_loc` was rebound to FULL-side - # KERNEL-FACING ids at ForwardBatch construction, so the full->swa - # map cannot be re-run on it -- those values index far past the swa - # v2p table (a device-side "index out of bounds" assert). Phase 2 of - # the write contract derives the swa loc from them instead. + # physical ids at ForwardBatch construction, so the full->swa map + # cannot be re-run on it: a full physical page number is also a + # valid virtual page of the swa v2p, so the map would silently + # resolve it to an unrelated token's swa slot. Phase 2 of the write + # contract derives the swa loc from them instead. swa_write_loc = ( self.kv_index_translator.sliding_window_write_loc_for( forward_batch.out_cache_loc diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index 2d6175f466..cd8f39742f 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -165,7 +165,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): # Full-attention transfers use virtual IDs. SWA transfers already use # kernel-facing IDs from translate_loc_from_full_to_swa. kvcache.full_kv_pool.host_transfer_translate = ( - self.full_attn_allocator.translate_kv_loc_for_kernel + self.full_attn_allocator.translate_kv_loc ) self.free_group = None @@ -287,8 +287,8 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): *, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """SWA-layer read path: virtual TOKEN ids -> swa kernel-facing ids.""" - return self.swa_attn_allocator.translate_kv_loc_for_kernel(kv_indices, out=out) + """SWA-layer read path: virtual TOKEN ids -> swa-physical TOKEN ids.""" + return self.swa_attn_allocator.translate_kv_loc(kv_indices, out=out) @property def full_v2p_page_table(self) -> torch.Tensor: @@ -316,14 +316,9 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): def translate_swa_indices_for_transfer( self, kv_indices: torch.Tensor ) -> torch.Tensor: - """Virtual TOKEN ids -> swa-sub-pool PHYSICAL token ids. - - The SWA counterpart of the above. `translate_loc_from_full_to_swa` - cannot serve here: it returns KERNEL-FACING ids (the physical page - scaled by the sub-pool's per-page block count), which index the - per-layer views, whereas the SWA state component is registered as whole - page envelopes and addressed by physical page. - """ + """Virtual TOKEN ids -> swa-sub-pool PHYSICAL token ids, for the PD + transfer engine: the same translate as `translate_loc_from_full_to_swa`, + on int64 ids.""" return self.swa_attn_allocator.translate_kv_loc(kv_indices.to(torch.int64)) def _move_gate_targets(self): @@ -377,15 +372,6 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): lazy_compaction=self.lazy_compaction, ) - def translate_kv_loc_for_kernel( - self, - loc: torch.Tensor, - *, - out: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Full-pool virtual TOKEN ids -> kernel-facing ids.""" - return self.full_attn_allocator.translate_kv_loc_for_kernel(loc, out=out) - def translate_write_loc_for_kernel( self, loc: torch.Tensor, diff --git a/python/sglang/srt/mem_cache/allocator/unified_mamba.py b/python/sglang/srt/mem_cache/allocator/unified_mamba.py index f00e8ed926..41d25e6f10 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_mamba.py +++ b/python/sglang/srt/mem_cache/allocator/unified_mamba.py @@ -129,7 +129,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): # HiCache indexes the full sub-pool's per-layer views with kernel-facing IDs. kvcache.full_kv_pool.host_transfer_translate = ( - self.full_attn_allocator.translate_kv_loc_for_kernel + self.full_attn_allocator.translate_kv_loc ) # -- size: dynamic -- @@ -255,8 +255,9 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): *, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Full-pool virtual TOKEN ids -> physical TOKEN ids; `-1` passes through as - `-1` (padding downstream). ``out=`` supports cuda-graph buffer stability.""" + """Full-pool virtual TOKEN ids -> physical TOKEN ids; an unmapped or + negative id lands on 0, the sink. ``out=`` supports cuda-graph buffer + stability.""" result = self.full_attn_allocator.translate_kv_loc(loc, out=out) return result @@ -272,15 +273,6 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): """Page-level physical->virtual table of the full sub-pool.""" return self.full_attn_allocator.physical_to_virtual - def translate_kv_loc_for_kernel( - self, - loc: torch.Tensor, - *, - out: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Full-pool virtual TOKEN ids -> kernel-facing ids.""" - return self.full_attn_allocator.translate_kv_loc_for_kernel(loc, out=out) - def translate_write_loc_for_kernel( self, loc: torch.Tensor, diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index c6a93965d3..01fd789a62 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -993,76 +993,21 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): *, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Translate token-granular virtual ids to physical ids. - - Under DCP the input is the DCP-collapsed id (`widened // dcp_size`, what - `KVIndexTranslator.translate_dcp_read_ids` hands down), so this works on - `pool_page_size`. ``out=`` writes in-place into a caller-owned buffer, - required under cuda-graph capture: the captured graph records the gather - against a fixed ``data_ptr``. - """ - if out is not None: - assert out.dtype == torch.int64, ( - f"translate_kv_loc: out= dtype must be int64 (matches v2p), " - f"got {out.dtype}" - ) - assert out.shape == virt_tokens.shape, ( - f"translate_kv_loc: out= shape {tuple(out.shape)} must match " - f"virt_tokens shape {tuple(virt_tokens.shape)}" - ) - with record_function("MultiEndedAlloc.translate_kv_loc"): - return self._translate_kv_loc_impl(virt_tokens, out) - - def _translate_kv_loc_impl( - self, - virt_tokens: torch.Tensor, - out: Optional[torch.Tensor], - ) -> torch.Tensor: - # Tombstone-safety clamp: a tombstoned v2p entry (-1) must not reach - # `k_buffer[-1]` (illegal access under captured graph replay). Clamping to - # 0 routes it to physical slot 0, reserved sink space holding no real data. - ps = self.pool_page_size - if ps == 1: - if out is not None: - # `index_select(out=out)` forbids index/out aliasing, but the - # canonical caller passes `out=kv_indices` in place. - tmp = torch.index_select(self.virtual_to_physical, 0, virt_tokens) - tmp = torch.clamp_min(tmp, 0) - out.copy_(tmp) - return out - result = torch.index_select(self.virtual_to_physical, 0, virt_tokens) - return torch.clamp_min(result, 0) - # ps > 1: page math. `virt_pages`/`offsets` are fresh, so they - # cannot alias `out` -- `index_select(out=out)` is safe. - virt_pages = virt_tokens // ps - offsets = virt_tokens % ps - if out is not None: - torch.index_select(self.virtual_to_physical, 0, virt_pages, out=out) - out.mul_(ps) - out.add_(offsets) - out.clamp_(min=0) # tombstoned page: -1*ps + offset in [-ps, -1] - return out - phys_pages = self.virtual_to_physical[virt_pages] - result = phys_pages * ps + offsets - return torch.clamp_min(result, 0) - - def translate_kv_loc_for_kernel( - self, - virt_tokens: torch.Tensor, - *, - out: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Virtual token ids -> kernel-facing ids, which under the token-major - views are the physical token ids: + """Virtual token ids -> physical token ids, which under the token-major + views ARE the kernel-facing ids: kernel_id(t) = v2p[t // ps] * ps + t % ps - Same id as `translate_kv_loc` for any mapped virtual token; this one - is one Triton launch, and it is the path that sends an unmapped or - negative loc to id 0, the page-0 sink. int64 out; a consumer whose - kernel ABI wants int32 narrows where it fills that buffer. + Under DCP the input is the DCP-collapsed id (`widened // dcp_size`, what + `KVIndexTranslator.translate_dcp_read_ids` hands down), so this works on + `pool_page_size`. An unmapped, out-of-range or negative id resolves to + 0, the page-0 sink every kernel skips. ``out=`` writes in-place into a + caller-owned buffer, required under cuda-graph capture: the captured + graph records the gather against a fixed ``data_ptr``. int64 out; a + consumer whose kernel ABI wants int32 narrows where it fills that + buffer. """ - with record_function("MultiEndedAlloc.translate_kv_loc_for_kernel"): + with record_function("MultiEndedAlloc.translate_kv_loc"): return self._translate_loc_fused(virt_tokens, dcp_size=1, out=out) def _translate_loc_fused( @@ -1075,17 +1020,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): out_width: Optional[int] = None, ) -> torch.Tensor: """One launch for the read and write conversions alike; see - `write_loc_to_kernel_ids`.""" - if out is not None: - assert out.dtype == torch.int64, ( - f"translate_kv_loc_for_kernel: out= dtype must be int64 (matches v2p), " - f"got {out.dtype}" - ) - if out_width is None: - assert out.shape == loc.shape, ( - f"translate_kv_loc_for_kernel: out= shape {tuple(out.shape)} must " - f"match virt_tokens shape {tuple(loc.shape)}" - ) + `write_loc_to_kernel_ids`, which owns the `out=` dtype and shape + contract for both.""" return write_loc_to_kernel_ids( loc=loc, v2p=self.virtual_to_physical, diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index d1d6a4b754..999e62fc2a 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -124,7 +124,7 @@ class KVIndexTranslator: self._capture_page_size = alloc.page_size self._full_v2p_table = alloc.full_v2p_page_table self._full_p2v_table = alloc.full_p2v_page_table - self._translate_full = alloc.translate_kv_loc_for_kernel + self._translate_full = alloc.translate_kv_loc # The WRITE loc is the one id that arrives DCP-WIDENED: read indices # are collapsed by the DCP index kernels, `out_cache_loc` still # carries the owner rule in `loc % dcp_size`. Identity with the read diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 7baf2c46f3..caf2be3e87 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -1595,12 +1595,12 @@ class UnifiedSWAKVPool(SWAKVPool): return # no-op in shared mode (the swa-side v2p IS the mapping) def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor): - """Virtual token ids -> swa kernel-facing ids (int64).""" + """Virtual token ids -> swa-physical token ids (int64).""" assert self._swa_allocator is not None, ( "UnifiedSWAKVPool.translate_loc_from_full_to_swa called before " "attach_allocators" ) - return self._swa_allocator.translate_kv_loc_for_kernel(kv_indices) + return self._swa_allocator.translate_kv_loc(kv_indices) def get_state_buf_infos(self): return self.swa_kv_pool.get_contiguous_buf_infos() diff --git a/test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py b/test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py new file mode 100644 index 0000000000..1c7e327b1f --- /dev/null +++ b/test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py @@ -0,0 +1,146 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""``write_loc_to_kernel_ids`` must walk `loc` and `out` by their own strides. +The SWA read path hands it a column slice of a capture-stable page table, whose +row stride is the whole buffer width; flat addressing reads the wrong ids there +without any error. Only this test enters the Triton kernel -- the CPU reference +never does. + + python -m pytest test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py -v +""" + +import itertools +import unittest + +import torch + +from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") +class TestWriteLocToKernelIdsStrided(unittest.TestCase): + ROWS, COLS, PAD = 6, 13, 11 + + def _v2p(self, num_pages): + # Scrambled, with one tombstone so a missing clamp shows up. + v2p = torch.tensor( + [(5 * i + 2) % num_pages for i in range(num_pages)] + [-1], + dtype=torch.int64, + device="cuda", + ) + v2p[3] = -1 + return v2p + + # Lanes that must resolve to the sink: padding, a tombstoned page, a loc + # below the sentinel, a page past the table. + SINK_LANES = (0, 7, 11, 12) + + def _locs(self, page_size, dtype=torch.int64): + """Every lane kind the kernel tells apart, as `(locs, num_pages)`: + non-zero in-page offsets, the -1 padding sink, a tombstoned page, a + loc below the sentinel and a page past a `num_pages` table.""" + n = self.ROWS * self.COLS + lane = torch.arange(n, dtype=torch.int64, device="cuda") + locs = lane * page_size + lane % page_size + locs[0] = -1 # the padding sink + locs[7] = 4 * page_size - 1 # the last slot of the tombstoned page 3 + num_pages = int(locs.max()) // page_size + 4 + locs[11] = -(page_size + 1) # below the sentinel + locs[12] = (num_pages + 2) * page_size + 1 # past the table + return locs.reshape(self.ROWS, self.COLS).to(dtype), num_pages + + def _run(self, loc, v2p, page_size, out=None): + return write_loc_to_kernel_ids(loc=loc, v2p=v2p, page_size=page_size, out=out) + + def test_a_column_slice_reads_the_same_ids_as_its_contiguous_copy(self): + for page_size, dtype in itertools.product((1, 64), (torch.int64, torch.int32)): + with self.subTest(page_size=page_size, dtype=dtype): + values, num_pages = self._locs(page_size, dtype) + v2p = self._v2p(num_pages) + backing = torch.full( + (self.ROWS, self.COLS + self.PAD), + -1, + dtype=dtype, + device="cuda", + ) + view = backing[:, : self.COLS] + view.copy_(values) + self.assertFalse(view.is_contiguous()) + + got = self._run(view, v2p, page_size) + want = self._run(view.contiguous(), v2p, page_size) + self.assertEqual(got.shape, view.shape) + self.assertTrue(torch.equal(got, want), f"page_size={page_size}") + + def test_a_column_stride_above_one_is_honoured(self): + page_size = 1 + values, num_pages = self._locs(page_size) + v2p = self._v2p(num_pages) + backing = torch.empty( + (self.ROWS, self.COLS * 2), dtype=torch.int64, device="cuda" + ) + view = backing.as_strided((self.ROWS, self.COLS), (self.COLS * 2, 2)) + view.copy_(values) + self.assertTrue( + torch.equal( + self._run(view, v2p, page_size), self._run(values, v2p, page_size) + ) + ) + + def test_out_takes_its_own_row_stride_and_leaves_its_neighbours_alone(self): + page_size = 64 + values, num_pages = self._locs(page_size) + v2p = self._v2p(num_pages) + # `out`'s row stride differs from `loc`'s, so one decomposition cannot + # serve both; each side must use its own. + loc_backing = torch.full( + (self.ROWS, self.COLS + 3), -1, dtype=torch.int64, device="cuda" + ) + loc = loc_backing[:, : self.COLS] + loc.copy_(values) + dst = torch.full( + (self.ROWS, self.COLS + self.PAD), -9, dtype=torch.int64, device="cuda" + ) + + ret = self._run(loc, v2p, page_size, out=dst[:, : self.COLS]) + self.assertEqual(ret.data_ptr(), dst.data_ptr()) + self.assertTrue( + torch.equal(dst[:, : self.COLS], self._run(values, v2p, page_size)) + ) + self.assertTrue(bool((dst[:, self.COLS :] == -9).all())) + + def test_the_cpu_reference_agrees_on_the_same_strided_input(self): + for page_size, dtype in itertools.product((1, 64), (torch.int64, torch.int32)): + with self.subTest(page_size=page_size, dtype=dtype): + values, num_pages = self._locs(page_size, dtype) + v2p = self._v2p(num_pages) + backing = torch.full( + (self.ROWS, self.COLS + self.PAD), -1, dtype=dtype, device="cuda" + ) + view = backing[:, : self.COLS] + view.copy_(values) + + cuda_ids = self._run(view, v2p, page_size) + cpu_backing = backing.cpu() + cpu_ids = self._run(cpu_backing[:, : self.COLS], v2p.cpu(), page_size) + self.assertTrue(torch.equal(cuda_ids.cpu(), cpu_ids)) + sink = cuda_ids.reshape(-1)[list(self.SINK_LANES)] + self.assertTrue(bool((sink == 0).all()), f"sink lanes: {sink}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_unified_memory_move_gate.py b/test/registered/unit/disaggregation/test_unified_memory_move_gate.py index e5b7a5d727..201d8eee20 100644 --- a/test/registered/unit/disaggregation/test_unified_memory_move_gate.py +++ b/test/registered/unit/disaggregation/test_unified_memory_move_gate.py @@ -316,9 +316,8 @@ class TestUnifiedAllocatorsPublishTheTransferContract(CustomTestCase): def test_swa_composite_translates_the_swa_side_separately(self): """The SWA sub-pool runs its OWN compaction, so a full-side physical id - does not name the SWA page holding the same virtual token. The read-path - `translate_loc_from_full_to_swa` cannot stand in either: it returns - kernel-facing ids, and the transfer addresses raw page envelopes.""" + does not name the SWA page holding the same virtual token: the transfer + translate must go through the SWA sub-pool's own table.""" virtual = torch.tensor([1, 3], dtype=torch.int32) for name in ( "UnifiedSWATokenToKVPoolAllocator", @@ -331,7 +330,6 @@ class TestUnifiedAllocatorsPublishTheTransferContract(CustomTestCase): ) alloc.swa_attn_allocator = SimpleNamespace( translate_kv_loc=lambda ids: ids + 32, - translate_kv_loc_for_kernel=lambda ids: ids + 64, ) physical = alloc.translate_swa_indices_for_transfer(virtual) self.assertEqual(physical.dtype, torch.int64) diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index 56c86fa7f6..08ce1fa013 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -2479,20 +2479,19 @@ class TestSWACompositeKernelIdSurface(unittest.TestCase): self.assertIs(a.full_v2p_page_table, a.full_attn_allocator.virtual_to_physical) self.assertIs(a.swa_v2p_page_table, a.swa_attn_allocator.virtual_to_physical) - def test_full_kernel_translate_matches_formula(self): - """With the per-page scale gone the two translates compute the same - id, so both must follow v2p[t // ps] * ps + t % ps.""" + def test_full_translate_matches_formula(self): a = self._build() v = a.alloc(3 * self.PS) self.assertIsNotNone(v) v2p = a.full_attn_allocator.virtual_to_physical expected = v2p[v // self.PS] * self.PS + v % self.PS - self.assertTrue(torch.equal(a.translate_kv_loc_for_kernel(v), expected)) self.assertTrue(torch.equal(a.translate_kv_loc(v), expected)) - def test_kernel_translate_accepts_an_int32_page_table(self): - """Regression: fa3 passes its own page table, which is int32 and 2-D, so - the gather must not require an int64 index.""" + def test_translate_accepts_an_int32_page_table(self): + """fa3 translates its own page table, which is int32 and 2-D; a gather + that needs a 1-D int64 index would crash the scheduler there. Both page + sizes: at ps == 1 the index IS the caller's tensor, at ps > 1 it is + derived.""" for ps in (1, 4): with self.subTest(page_size=ps): self.PS = ps @@ -2502,12 +2501,12 @@ class TestSWACompositeKernelIdSurface(unittest.TestCase): v2p = a.full_attn_allocator.virtual_to_physical expected = v2p[v // ps] * ps + v % ps page_table = v.to(torch.int32).view(2, -1) - got = a.translate_kv_loc_for_kernel(page_table) + got = a.translate_kv_loc(page_table) self.assertEqual(got.shape, page_table.shape) self.assertTrue(torch.equal(got.reshape(-1), expected)) # `out=` takes the same int32 index; the buffer stays int64. dst = torch.empty(page_table.shape, dtype=torch.int64, device=_DEV) - a.translate_kv_loc_for_kernel(page_table, out=dst) + a.translate_kv_loc(page_table, out=dst) self.assertTrue(torch.equal(dst.reshape(-1), expected)) def test_swa_translate_matches_formula(self): @@ -2542,7 +2541,7 @@ class TestSWACompositeKernelIdSurface(unittest.TestCase): physical_pages.tolist(), ) - def test_swa_kernel_tombstone_still_lands_on_sink(self): + def test_swa_tombstone_still_lands_on_sink(self): """A tombstoned page's ids (v2p == -1 -> -ps + offset, negative for every in-page offset) still land on the sink, never negative.""" a = self._build() @@ -2626,7 +2625,7 @@ class TestPs64MLACompositeFeasibility(unittest.TestCase): # fits int32 (the canonical narrows on store). v2p = a.full_v2p_page_table want = v2p[v // self.PS] * self.PS + v % self.PS - got = a.translate_kv_loc_for_kernel(v) + got = a.translate_kv_loc(v) self.assertTrue(torch.equal(got, want), "kernel-facing formula broke at ps=64") self.assertTrue(bool((got < 2**31).all().item())) @@ -3256,7 +3255,7 @@ class TestDcpWidening(unittest.TestCase): self.assertTrue( torch.equal( written[owned], - a.translate_kv_loc_for_kernel(ids[owned] // dcp_size), + a.translate_kv_loc(ids[owned] // dcp_size), ) ) # ...and the rest go to the sink the write kernels skip. @@ -3604,6 +3603,52 @@ class TestFusedWriteLocTranslate(unittest.TestCase): device="cpu", ) + def test_matches_reference_on_a_strided_view(self): + """A column slice of a wider page table is what the SWA read path hands + down under cuda graphs, so its ids must equal the flat reference's.""" + from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + + rows, cols, page_size = 3, 5, 4 + v2p = torch.arange(32, dtype=torch.int64, device=_DEV) + backing = torch.full((rows, cols + 3), -1, dtype=torch.int64, device=_DEV) + view = backing[:, :cols] + view.copy_(torch.arange(rows * cols, dtype=torch.int64).reshape(rows, cols)) + self.assertFalse(view.is_contiguous()) + + want = self._reference(view.reshape(-1).cpu(), v2p.cpu(), page_size, 1, 0) + got = write_loc_to_kernel_ids(loc=view, v2p=v2p, page_size=page_size) + self.assertEqual(got.shape, view.shape) + self.assertEqual(got.reshape(-1).tolist(), want) + + # `out=` into a slice of a DIFFERENTLY strided backing: the ids land in + # the sliced columns and the rest of that buffer is left alone. + dst = torch.full((rows, cols + 7), -9, dtype=torch.int64, device=_DEV) + ret = write_loc_to_kernel_ids( + loc=view, + v2p=v2p, + page_size=page_size, + out=dst[:, :cols], + ) + self.assertEqual(ret.data_ptr(), dst.data_ptr()) + self.assertEqual(dst[:, :cols].reshape(-1).tolist(), want) + self.assertTrue(bool((dst[:, cols:] == -9).all())) + + def test_out_width_needs_a_packed_loc(self): + """`out_width` addresses a packed lane range, so a strided loc would + write its tail zeros over live ids.""" + from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + + v2p = torch.arange(16, dtype=torch.int64, device=_DEV) + loc = torch.arange(12, dtype=torch.int64, device=_DEV)[::2] + with self.assertRaises(AssertionError): + write_loc_to_kernel_ids( + loc=loc, + v2p=v2p, + page_size=1, + out=torch.zeros(10, dtype=torch.int64, device=_DEV), + out_width=10, + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_mla_views.py b/test/registered/unit/mem_cache/test_unified_mla_views.py index 378c7fdae1..87c08fbb5b 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_views.py +++ b/test/registered/unit/mem_cache/test_unified_mla_views.py @@ -311,7 +311,7 @@ class _FakeKVCache: self.buf[dst_loc] = self.buf[src_loc].clone() -class TestTranslateKvLocForKernel(unittest.TestCase): +class TestTranslateKvLoc(unittest.TestCase): def _build(self, ps=1, n_full_tokens=64): pool, full, mamba = _make_unified(page_size=ps, n_full_tokens=n_full_tokens) full_alloc = MultiEndedAllocator( @@ -333,39 +333,82 @@ class TestTranslateKvLocForKernel(unittest.TestCase): mamba_alloc.bind_peer(full_alloc) return full_alloc - def test_kernel_id_is_the_physical_id(self): + def test_translate_matches_v2p_formula(self): for ps in (1, 4): alloc = self._build(ps=ps) v = alloc.alloc(3 * ps) self.assertIsNotNone(v) - phys = alloc.translate_kv_loc(v) - kernel = alloc.translate_kv_loc_for_kernel(v) - self.assertTrue(torch.equal(kernel, phys), f"ps={ps}") v2p = alloc.virtual_to_physical - self.assertTrue(torch.equal(kernel, v2p[v // ps] * ps + v % ps)) + want = v2p[v // ps] * ps + v % ps + self.assertTrue(torch.equal(alloc.translate_kv_loc(v), want), f"ps={ps}") def test_tombstone_clamps_to_sink(self): alloc = self._build(ps=1) # never-allocated virtual ids -> v2p == -1 -> id 0 virt = torch.tensor([alloc.min_slot_index + 1], dtype=torch.int64) - kernel = alloc.translate_kv_loc_for_kernel(virt) - self.assertTrue(torch.all(kernel == 0)) + self.assertTrue(torch.all(alloc.translate_kv_loc(virt) == 0)) def test_out_matches_and_aliases(self): for ps in (1, 4): alloc = self._build(ps=ps) v = alloc.alloc(2 * ps) self.assertIsNotNone(v) - no_out = alloc.translate_kv_loc_for_kernel(v) + no_out = alloc.translate_kv_loc(v) out = torch.empty_like(v) - ret = alloc.translate_kv_loc_for_kernel(v, out=out) + ret = alloc.translate_kv_loc(v, out=out) self.assertIs(ret, out) self.assertTrue(torch.all(out == no_out)) # canonical in-place aliasing: translate(x, out=x) x = v.clone() - alloc.translate_kv_loc_for_kernel(x, out=x) + alloc.translate_kv_loc(x, out=x) self.assertTrue(torch.all(x == no_out)) + def test_padding_loc_lands_on_the_sink(self): + """A padded read table carries -1 in the slots a shorter sequence does + not use; `translate_kv_loc` resolves it to 0, the sink.""" + for ps in (1, 4): + alloc = self._build(ps=ps) + self.assertIsNotNone(alloc.alloc(4 * ps)) + got = alloc.translate_kv_loc(torch.tensor([-1], dtype=torch.int64)) + self.assertTrue(bool((got == 0).all()), f"ps={ps}: {got}") + + def test_a_loc_past_the_sentinel_lands_on_the_sink(self): + """Only `-1` reaches the trailing v2p sentinel; a loc below it floors + onto a REAL page, so a translate that merely indexes with it hands back + a live slot belonging to another request.""" + for ps in (1, 4): + alloc = self._build(ps=ps) + self.assertIsNotNone(alloc.alloc(4 * ps)) + alloc.virtual_to_physical[-2] = 7 # bind the last real page + got = alloc.translate_kv_loc(torch.tensor([-(ps + 1)], dtype=torch.int64)) + self.assertTrue(bool((got == 0).all()), f"ps={ps}: {got}") + + def test_an_out_of_range_loc_lands_on_the_sink(self): + """An id past the v2p table resolves to the sink instead of reading + out of bounds.""" + for ps in (1, 4): + alloc = self._build(ps=ps) + self.assertIsNotNone(alloc.alloc(4 * ps)) + past = int(alloc.virtual_to_physical.numel()) * ps * 4 + got = alloc.translate_kv_loc(torch.tensor([past], dtype=torch.int64)) + self.assertTrue(bool((got == 0).all()), f"ps={ps}: {got}") + + def test_translate_accepts_a_strided_page_table(self): + """The SWA read path hands down `page_table[:bs, :max_seq_len]`, a + column slice of the capture-stable buffer.""" + for ps in (1, 4): + alloc = self._build(ps=ps) + v = alloc.alloc(4 * ps) + self.assertIsNotNone(v) + want = alloc.translate_kv_loc(v) + backing = torch.full((2, 2 * v.numel()), -1, dtype=torch.int64) + view = backing[:, : v.numel() // 2] + view.copy_(v.view(2, -1)) + self.assertFalse(view.is_contiguous()) + got = alloc.translate_kv_loc(view) + self.assertEqual(got.shape, view.shape) + self.assertTrue(torch.equal(got.reshape(-1), want)) + class _RecordingHybridPool: """Stands in for `UnifiedHybridLinearKVPool`, recording the ids it is handed."""