From d3dbb71e5a6111cb09a9a5105aa3b6317d9b229e Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:07:47 -0700 Subject: [PATCH] [unified-memory] Derive KV row addresses from strides, not shapes (1/7) (#40326) Co-authored-by: Cheng Wan --- .../csrc/attention/fused_fp8_qkv_kv_cache.cuh | 19 +- .../inkling/inkling_attn_prologue_fused.cuh | 2 +- .../jit/csrc/kv_canary/canary_common.cuh | 6 +- .../jit/csrc/kv_canary/canary_verify.cuh | 6 +- .../jit/csrc/kv_canary/canary_write.cuh | 6 +- python/sglang/kernels/ops/attention/utils.py | 33 ++-- python/sglang/kernels/ops/kv_canary/verify.py | 30 +-- .../kv_canary/pool_patcher/buffer_alloc.py | 5 +- python/sglang/srt/mem_cache/memory_pool.py | 36 ++-- python/sglang/srt/mem_cache/pool_host/base.py | 48 +++++ python/sglang/srt/mem_cache/pool_host/mha.py | 33 +++- python/sglang/srt/mem_cache/pool_host/mla.py | 32 +++- .../sglang/srt/models/inkling_common/attn.py | 95 +++++----- .../test/kernels/kv_canary/_fixtures.py | 13 +- .../kernels/ops/kv_canary/test_verify_hand.py | 41 +++++ .../kernels/ops/kv_canary/test_write_hand.py | 27 +++ .../kvcache/test_fused_fp8_qkv_kv_cache.py | 43 +++++ .../test_hicache_strided_device_rows.py | 95 ++++++++++ .../test_masked_set_kv_buffer_strided.py | 82 +++++++++ .../test_asymmetric_mha_pool_host_unit.py | 28 +++ .../unit/mem_cache/test_kv_stride_hygiene.py | 171 ++++++++++++++++++ 21 files changed, 733 insertions(+), 118 deletions(-) create mode 100644 test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py create mode 100644 test/registered/kernels/ops/kvcache/test_masked_set_kv_buffer_strided.py create mode 100644 test/registered/unit/mem_cache/test_kv_stride_hygiene.py diff --git a/python/sglang/kernels/jit/csrc/attention/fused_fp8_qkv_kv_cache.cuh b/python/sglang/kernels/jit/csrc/attention/fused_fp8_qkv_kv_cache.cuh index 221e13fbd7..c171abe444 100644 --- a/python/sglang/kernels/jit/csrc/attention/fused_fp8_qkv_kv_cache.cuh +++ b/python/sglang/kernels/jit/csrc/attention/fused_fp8_qkv_kv_cache.cuh @@ -24,6 +24,9 @@ struct FusedQkvParams { int64_t q_stride; int64_t k_stride; int64_t v_stride; + // Elements between consecutive cache slots; may exceed kv_dim (strided per-layer views). + int64_t k_cache_stride; + int64_t v_cache_stride; uint32_t num_tokens; uint32_t q_dim; uint32_t kv_dim; @@ -76,12 +79,12 @@ __global__ void fused_fp8_qkv_kv_cache_kernel(const __grid_constant__ FusedQkvPa } quant_row( static_cast(params.k) + static_cast(token) * params.k_stride, - static_cast(params.k_cache) + static_cast(slot) * params.kv_dim, + static_cast(params.k_cache) + static_cast(slot) * params.k_cache_stride, params.kv_dim, inv_k); quant_row( static_cast(params.v) + static_cast(token) * params.v_stride, - static_cast(params.v_cache) + static_cast(slot) * params.kv_dim, + static_cast(params.v_cache) + static_cast(slot) * params.v_cache_stride, params.kv_dim, inv_v); @@ -127,13 +130,16 @@ struct FusedFp8QkvKvCache { auto S = SymbolicSize{"num_slots"}; auto SK = SymbolicSize{"k_stride"}; auto SV = SymbolicSize{"v_stride"}; + auto SKC = SymbolicSize{"k_cache_stride"}; + auto SVC = SymbolicSize{"v_cache_stride"}; auto device = SymbolicDevice{}; auto idx_dtype = SymbolicDType{}; device.set_options(); TensorMatcher({N, Dkv}).with_strides({SK, 1}).with_dtype().with_device(device).verify(k); TensorMatcher({N, Dkv}).with_strides({SV, 1}).with_dtype().with_device(device).verify(v); - TensorMatcher({S, Dkv}).with_dtype().with_device(device).verify(k_cache).verify(v_cache); + TensorMatcher({S, Dkv}).with_strides({SKC, 1}).with_dtype().with_device(device).verify(k_cache); + TensorMatcher({S, Dkv}).with_strides({SVC, 1}).with_dtype().with_device(device).verify(v_cache); TensorMatcher({N}).with_dtype(idx_dtype).with_device(device).verify(cache_loc); TensorMatcher({1}).with_dtype().with_device(device).verify(k_scale).verify(v_scale); @@ -156,12 +162,17 @@ struct FusedFp8QkvKvCache { const uint32_t kv_dim = static_cast(Dkv.unwrap()); const int64_t k_stride = SK.unwrap(); const int64_t v_stride = SV.unwrap(); + const int64_t k_cache_stride = SKC.unwrap(); + const int64_t v_cache_stride = SVC.unwrap(); RuntimeCheck(num_tokens > 0, "fused_fp8_qkv_kv_cache: num_tokens must be > 0, got ", num_tokens); auto fits = [&](int vec) { const int in_bytes = vec * static_cast(sizeof(T)); + const int out_bytes = vec * static_cast(sizeof(fp8_e4m3_t)); bool ok = kv_dim % vec == 0 && k_stride % vec == 0 && v_stride % vec == 0 && aligned(k.data_ptr(), in_bytes) && aligned(v.data_ptr(), in_bytes); + ok = ok && k_cache_stride % vec == 0 && v_cache_stride % vec == 0 && aligned(k_cache.data_ptr(), out_bytes) && + aligned(v_cache.data_ptr(), out_bytes); if (quantize_q) { ok = ok && q_dim % vec == 0 && q_stride % vec == 0 && aligned(q_ptr, in_bytes); } @@ -182,6 +193,8 @@ struct FusedFp8QkvKvCache { .q_stride = q_stride, .k_stride = k_stride, .v_stride = v_stride, + .k_cache_stride = k_cache_stride, + .v_cache_stride = v_cache_stride, .num_tokens = num_tokens, .q_dim = q_dim, .kv_dim = kv_dim, diff --git a/python/sglang/kernels/jit/csrc/inkling/inkling_attn_prologue_fused.cuh b/python/sglang/kernels/jit/csrc/inkling/inkling_attn_prologue_fused.cuh index 2adf03105e..c76cf94dfb 100644 --- a/python/sglang/kernels/jit/csrc/inkling/inkling_attn_prologue_fused.cuh +++ b/python/sglang/kernels/jit/csrc/inkling/inkling_attn_prologue_fused.cuh @@ -132,7 +132,7 @@ struct AttnPrologueParams { int64_t inter_stride_b; int64_t inter_stride_t; int64_t inter_stride_w; - int64_t kv_buf_stride; // elems per KV slot row (= Hkv * head_dim) + int64_t kv_buf_stride; // elems between consecutive KV slots (>= Hkv * head_dim) uint32_t T; uint32_t q; // draft_token_num uint32_t dq; diff --git a/python/sglang/kernels/jit/csrc/kv_canary/canary_common.cuh b/python/sglang/kernels/jit/csrc/kv_canary/canary_common.cuh index 565f25b2d7..14e417b01a 100644 --- a/python/sglang/kernels/jit/csrc/kv_canary/canary_common.cuh +++ b/python/sglang/kernels/jit/csrc/kv_canary/canary_common.cuh @@ -12,7 +12,7 @@ namespace canary { // Device-side handle for one real-KV source. struct RealKvSourceHandle { const uint8_t* tensor; // raw uint8 byte pointer to the source tensor - int32_t row_stride_bytes; // tensor.shape[1] in bytes (may exceed page_size * num_bytes_per_token) + int32_t row_stride_bytes; // tensor.stride(0) in bytes (may exceed page_size * num_bytes_per_token) int32_t page_size; int32_t num_bytes_per_token; int32_t read_bytes; @@ -37,8 +37,8 @@ SGL_DEVICE uint64_t splitmix64_mix3(uint64_t a, uint64_t b, uint64_t c) { // tensor[slot_idx // page_size, // (slot_idx % page_size) * num_bytes_per_token + byte_offset] // -// row_stride_bytes is the dim-1 size of the underlying tensor in bytes (which may exceed -// page_size * num_bytes_per_token; trailing bytes are skipped). +// row_stride_bytes is the dim-0 stride of the underlying tensor in bytes (which may exceed +// page_size * num_bytes_per_token; the remaining bytes are skipped). SGL_DEVICE void real_kv_load_uint4( const RealKvSourceHandle& src, int64_t slot_idx, int64_t byte_offset, uint64_t& word_lo, uint64_t& word_hi) { const int64_t row = slot_idx / src.page_size; diff --git a/python/sglang/kernels/jit/csrc/kv_canary/canary_verify.cuh b/python/sglang/kernels/jit/csrc/kv_canary/canary_verify.cuh index dbb57729a1..c9cfd5476f 100644 --- a/python/sglang/kernels/jit/csrc/kv_canary/canary_verify.cuh +++ b/python/sglang/kernels/jit/csrc/kv_canary/canary_verify.cuh @@ -205,24 +205,28 @@ struct CanaryVerifyKernel { SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"}; SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"}; TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_0); SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"}; SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"}; TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_1); SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"}; SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"}; TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_2); SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"}; SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"}; TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_3); @@ -271,7 +275,7 @@ struct CanaryVerifyKernel { tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3}; for (int s = 0; s < kMaxRealKvSources; ++s) { p.sources[s].tensor = static_cast(source_bufs[s].data_ptr()); - p.sources[s].row_stride_bytes = static_cast(source_bufs[s].size(1)); + p.sources[s].row_stride_bytes = static_cast(source_bufs[s].stride(0)); p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize]; p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken]; p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes]; diff --git a/python/sglang/kernels/jit/csrc/kv_canary/canary_write.cuh b/python/sglang/kernels/jit/csrc/kv_canary/canary_write.cuh index 599e190ccd..eb5b6683d9 100644 --- a/python/sglang/kernels/jit/csrc/kv_canary/canary_write.cuh +++ b/python/sglang/kernels/jit/csrc/kv_canary/canary_write.cuh @@ -251,24 +251,28 @@ inline void canary_write_step_cuda( SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"}; SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"}; TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_0); SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"}; SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"}; TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_1); SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"}; SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"}; TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_2); SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"}; SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"}; TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3}) + .with_strides({-1, 1}) .with_dtype() .with_device(device_) .verify(real_kv_buf_3); @@ -329,7 +333,7 @@ inline void canary_write_step_cuda( tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3}; for (int s = 0; s < kMaxRealKvSources; ++s) { p.sources[s].tensor = static_cast(source_bufs[s].data_ptr()); - p.sources[s].row_stride_bytes = static_cast(source_bufs[s].size(1)); + p.sources[s].row_stride_bytes = static_cast(source_bufs[s].stride(0)); p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize]; p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken]; p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes]; diff --git a/python/sglang/kernels/ops/attention/utils.py b/python/sglang/kernels/ops/attention/utils.py index 3833847b5b..2bede6eb97 100644 --- a/python/sglang/kernels/ops/attention/utils.py +++ b/python/sglang/kernels/ops/attention/utils.py @@ -70,30 +70,21 @@ if _is_cuda: # # See: https://github.com/flashinfer-ai/flashinfer/issues/2232 def canonicalize_stride(tensor: torch.Tensor) -> torch.Tensor: - """ - Adjust degenerate strides for a tensor, make it canonical. + """Give each size-1 dim whose stride collides with the next dim's a + canonical stride; every other stride is kept, so a non-contiguous KV view + keeps its addressing. + + Example: shape ``[num_pages, 1, 64, 128]`` with stride + ``[8192, 128, 128, 1]`` becomes stride ``[8192, 8192, 128, 1]``. """ sizes = tensor.size() - strides = tensor.stride() - ndim = tensor.dim() - - need_fix = any( - sizes[i] == 1 and strides[i] == strides[i + 1] for i in range(ndim - 1) - ) - - if not need_fix: + strides = list(tensor.stride()) + new_strides = list(strides) + for i in range(tensor.dim() - 2, -1, -1): + if sizes[i] == 1 and strides[i] == strides[i + 1]: + new_strides[i] = new_strides[i + 1] * sizes[i + 1] + if new_strides == strides: return tensor - - # canonicalize the stride - # Example: - # - shape: [num_pages, 1, 64, 128] - # - stride: [8192, 128, 128, 1] (wrong!) - # Gives new stride: [8192, 8192, 128 ,1] (correct!) - new_strides = [0] * ndim - new_strides[-1] = 1 - for i in range(ndim - 2, -1, -1): - new_strides[i] = new_strides[i + 1] * sizes[i + 1] - return tensor.as_strided(sizes, new_strides) diff --git a/python/sglang/kernels/ops/kv_canary/verify.py b/python/sglang/kernels/ops/kv_canary/verify.py index 38de70a1c8..75a327466e 100644 --- a/python/sglang/kernels/ops/kv_canary/verify.py +++ b/python/sglang/kernels/ops/kv_canary/verify.py @@ -56,22 +56,23 @@ class RealKvSource: num_bytes_per_token``. Trailing bytes of each row are ignored by the canary; this is exactly how the abstraction accommodates pools whose per-row layout interleaves canary-relevant bytes with other metadata (layer-split storage, K/V interleaving, ...). When ``page_size == 1`` the pattern - collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case. + collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case. Rows are stepped by + ``tensor.stride(0)``, so dim 0 may be a per-layer view into a larger buffer. A pool may expose multiple RealKvSource instances per (canary buffer × K/V half) — the launch wrappers iterate the source list and fold each into the running real_kv_hash via splitmix64 (one int64 fingerprint per slot, regardless of source count). Pool patchers construct sources by: - - viewing / reshaping the underlying KV layer into the canonical [num_rows, dim1_bytes] form (no stage-copy - needed when the underlying storage is already row-major contiguous on dim 0), + - viewing the underlying KV layer as [num_rows, dim1_bytes] (a view, never a stage-copy: a copy would + fingerprint a stale snapshot instead of the live pool), - choosing ``page_size`` and ``num_bytes_per_token`` so that the access pattern above lands on the bytes the canary should fingerprint, - leaving any per-row padding / non-canary bytes in the trailing portion of each row (they will simply be skipped). 16-byte alignment precondition: the CUDA fold kernel issues 128-bit aligned loads, so ``read_bytes``, - ``num_bytes_per_token``, and the row stride (``tensor.shape[1]`` in bytes) must all be positive + ``num_bytes_per_token``, and the row stride (``tensor.stride(0)`` in bytes) must all be positive multiples of 16. There is no "skip this source" sentinel — callers omit the source from their ``real_kv_sources`` tuple entirely (factory helpers return an empty tuple in that case). @@ -114,19 +115,21 @@ class RealKvSource: raise ValueError( f"kv-canary: RealKvSource.tensor must be at least 2-D, got shape {tuple(self.tensor.shape)}" ) - row_stride_bytes = int(self.tensor.shape[1]) * self.tensor.element_size() + row_stride_bytes = int(self.tensor.stride(0)) * self.tensor.element_size() if row_stride_bytes % 16 != 0: raise ValueError( - f"kv-canary: RealKvSource.tensor dim-1 byte width must be a multiple of 16, " + f"kv-canary: RealKvSource.tensor row stride must be a multiple of 16 bytes, " f"got {row_stride_bytes} bytes (shape={tuple(self.tensor.shape)}, " - f"dtype={self.tensor.dtype})" + f"strides={tuple(self.tensor.stride())}, dtype={self.tensor.dtype})" ) # A row is addressed as page_size slots of num_bytes_per_token, unchecked at fold time; - # a narrower row hashes fewer bytes than asked and still reports the chain clean. + # a narrower row hashes fewer bytes than asked and still reports the chain clean. Check the + # row's width, not its stride. + row_bytes = int(self.tensor.shape[1]) * self.tensor.element_size() min_row_bytes = self.page_size * self.num_bytes_per_token - if row_stride_bytes < min_row_bytes: + if row_bytes < min_row_bytes: raise ValueError( - f"kv-canary: RealKvSource.tensor dim-1 is {row_stride_bytes} bytes but " + f"kv-canary: RealKvSource.tensor row is {row_bytes} bytes wide but " f"page_size={self.page_size} x num_bytes_per_token={self.num_bytes_per_token} " f"needs {min_row_bytes} (shape={tuple(self.tensor.shape)}, " f"dtype={self.tensor.dtype})" @@ -412,7 +415,12 @@ def _build_real_kv_source_abi( ) for i, source in enumerate(real_kv_sources): - _assert_contiguous(source.tensor, f"real_kv_sources[{i}].tensor") + # The kernels step rows by stride(0), so only the bytes within a row must be packed. + if source.tensor.stride(-1) != 1: + raise ValueError( + f"kv-canary: real_kv_sources[{i}].tensor rows must be packed " + f"(stride(-1) == 1), got strides {tuple(source.tensor.stride())}" + ) source_u8 = source.tensor.view(torch.uint8) if source_u8.dim() != 2: raise ValueError( diff --git a/python/sglang/srt/kv_canary/pool_patcher/buffer_alloc.py b/python/sglang/srt/kv_canary/pool_patcher/buffer_alloc.py index eb11ae5253..aed31a4c88 100644 --- a/python/sglang/srt/kv_canary/pool_patcher/buffer_alloc.py +++ b/python/sglang/srt/kv_canary/pool_patcher/buffer_alloc.py @@ -67,11 +67,10 @@ def make_row_source( layer_buffer: torch.Tensor, read_bytes: int, ) -> Tuple[RealKvSource, ...]: - contiguous = layer_buffer.contiguous() - num_slots = int(contiguous.shape[0]) + num_slots = int(layer_buffer.shape[0]) if num_slots == 0 or read_bytes == 0: return () - flat = contiguous.view(torch.uint8).reshape(num_slots, -1) + flat = layer_buffer.view(torch.uint8).view(num_slots, -1) num_bytes_per_token = int(flat.shape[1]) clipped = _clip_read_bytes_aligned( requested=read_bytes, num_bytes_per_token=num_bytes_per_token diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 0007fdacab..746439bb34 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -2117,6 +2117,13 @@ class MHATokenToKVPool(KVCache): self._kv_copy_config = None return + # The tiled copy uses `data_strides[i]` as both slot stride and copy width. + for buf in self._slot_move_pointer_buffers(): + assert buf.is_contiguous(), ( + "the tiled KV copy needs contiguous KV buffers; got shape " + f"{tuple(buf.shape)}, strides {buf.stride()}" + ) + # Heuristics for KV copy tiling _KV_COPY_STRIDE_THRESHOLD_LARGE = 8192 _KV_COPY_STRIDE_THRESHOLD_MEDIUM = 4096 @@ -2292,10 +2299,7 @@ class MHATokenToKVPool(KVCache): device=self.device, ) self.data_strides = torch.tensor( - [ - np.prod(x.shape[1:]) * x.dtype.itemsize - for x in slot_move_pointer_buffers - ], + [x.stride(0) * x.dtype.itemsize for x in slot_move_pointer_buffers], device=self.device, ) @@ -2647,11 +2651,13 @@ class MHATokenToKVPool(KVCache): if dcp_kv_mask is not None: N, H, D = cache_k.shape + k_buf = self.k_buffer[layer_id - self.start_layer] + v_buf = self.v_buffer[layer_id - self.start_layer] masked_set_kv_buffer_kernel[(N,)]( cache_k, cache_v, - self.k_buffer[layer_id - self.start_layer], - self.v_buffer[layer_id - self.start_layer], + k_buf, + v_buf, loc, dcp_kv_mask, N, @@ -2662,6 +2668,8 @@ class MHATokenToKVPool(KVCache): cache_k.stride(1), cache_v.stride(0), cache_v.stride(1), + k_buf.stride(0), + v_buf.stride(0), ) return @@ -3224,10 +3232,7 @@ class NoOpMHATokenToKVPool(MHATokenToKVPool): ) self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0) self.data_strides = torch.tensor( - [ - np.prod(x.shape[1:]) * x.dtype.itemsize - for x in self.k_buffer + self.v_buffer - ], + [x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer], device=self.device, ) @@ -3610,10 +3615,7 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool): ) self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0) self.data_strides = torch.tensor( - [ - np.prod(x.shape[1:]) * x.dtype.itemsize - for x in self.k_buffer + self.v_buffer - ], + [x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer], device=self.device, ) # This override replaces the base allocation, so the PD-transfer @@ -5295,6 +5297,8 @@ def masked_set_kv_buffer_kernel( k_stride_H: tl.constexpr, v_stride_B: tl.constexpr, v_stride_H: tl.constexpr, + k_buffer_stride: tl.constexpr, + v_buffer_stride: tl.constexpr, ): pid = tl.program_id(0) if pid >= N: @@ -5316,10 +5320,10 @@ def masked_set_kv_buffer_kernel( col = idx % D key = tl.load(k_ptr + pid * k_stride_B + row * k_stride_H + col, mask=mask) - tl.store(k_buffer_ptr + loc * H * D + idx, key, mask=mask) + tl.store(k_buffer_ptr + loc * k_buffer_stride + idx, key, mask=mask) value = tl.load(v_ptr + pid * v_stride_B + row * v_stride_H + col, mask=mask) - tl.store(v_buffer_ptr + loc * H * D + idx, value, mask=mask) + tl.store(v_buffer_ptr + loc * v_buffer_stride + idx, value, mask=mask) class MHATokenToKOnlyPool(KVCache): diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index 972b1cb6a1..826c754bba 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -278,6 +278,54 @@ class HostKVCache(abc.ABC): _cuda_host_unregister(buf) self.kv_buffer = None + # Bytes between strided device KV rows; None (the default, and what a pool + # built without `_init_device_row_stride` keeps) means packed rows. + _strided_device_row_bytes: Optional[int] = None + + def _init_device_row_stride(self, device_buffers: Iterable[torch.Tensor]) -> None: + """Record the row stride of strided device KV buffers. + + A strided per-layer view -- the unified pool's token-major entries -- + steps its rows by `stride(0)` rather than by the row width, and the + device-side addresses in the transfer kernels must follow it. + """ + buffers = list(device_buffers) + strides = { + buf.stride(0) * buf.element_size() + for buf in buffers + if not buf.is_contiguous() + } + if not strides: + return + if len(strides) > 1 or any(buf.is_contiguous() for buf in buffers): + raise NotImplementedError( + "HiCache: the device KV buffers do not share one row stride " + f"(strided rows {sorted(strides)} B, mixed with packed ones)" + ) + for buf in buffers: + assert buf.stride(-1) == 1, ( + f"HiCache needs packed device rows; got strides {buf.stride()}" + ) + self._strided_device_row_bytes = strides.pop() + + @property + def device_rows_packed(self) -> bool: + return self._strided_device_row_bytes is None + + @property + def device_row_stride_bytes(self) -> int: + """Bytes between consecutive device KV rows.""" + return self._strided_device_row_bytes or self.token_stride_size + + def _require_packed_device_rows(self, kernel: str) -> None: + if not self.device_rows_packed: + raise NotImplementedError( + f"HiCache: {kernel} steps device rows by the row width " + f"({self.token_stride_size} B), but this device pool's rows are " + f"{self.device_row_stride_bytes} B apart. Use the JIT HiCache " + "kernels with a layer_first or page_first host layout." + ) + @abc.abstractmethod def get_size_per_token(self): raise NotImplementedError() diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index 198a15a735..18b343b0c8 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -152,6 +152,13 @@ class MHATokenToKVPoolHost(HostKVCache): pool_label=pool_label, ) self.element_dim = self.head_num * self.head_dim + self._init_device_row_stride( + buf + for pool in (self.device_pool, *self.mtp_draft_device_pools) + if pool is not None + for side in ("k_buffer", "v_buffer") + for buf in getattr(pool, side, None) or () + ) # The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only # helpers in hicache.cuh are guarded by USE_ROCM and the staged # write-back kernel has a ROCm path, so enable them on HIP too. This @@ -268,7 +275,8 @@ class MHATokenToKVPoolHost(HostKVCache): self.staging_k_buffer = None self.staging_v_buffer = None self.can_use_write_back_jit = False - if self.layout != "page_first": + # The staged kernel reads whole device pages, so it needs packed rows. + if self.layout != "page_first" or not self.device_rows_packed: return page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK) staging = prepare_mha_write_back_staging( @@ -328,6 +336,7 @@ class MHATokenToKVPoolHost(HostKVCache): element_dim=self.element_dim, ) else: + self._require_packed_device_rows("transfer_kv_per_layer") transfer_kv_per_layer( src_k=self.k_buffer[host_layer_id], dst_k=device_pool.k_buffer[device_layer_id], @@ -353,6 +362,7 @@ class MHATokenToKVPoolHost(HostKVCache): element_dim=self.element_dim, ) else: + self._require_packed_device_rows("transfer_kv_per_layer_pf_lf") transfer_kv_per_layer_pf_lf( src_k=self.k_buffer, dst_k=device_pool.k_buffer[device_layer_id], @@ -365,6 +375,7 @@ class MHATokenToKVPoolHost(HostKVCache): src_layout_dim=self.layout_dim, ) elif self.layout == "page_head": + self._require_packed_device_rows("transfer_kv_per_layer_ph_lf") transfer_kv_per_layer_ph_lf( src_k=self.k_buffer, dst_k=device_pool.k_buffer[device_layer_id], @@ -489,10 +500,11 @@ class MHATokenToKVPoolHost(HostKVCache): v_ptr_src=device_v_data_ptrs, indices_src=device_indices, kv_cache_dst_stride_bytes=self.token_stride_size, - kv_cache_src_stride_bytes=self.token_stride_size, + kv_cache_src_stride_bytes=self.device_row_stride_bytes, element_size=self.element_dim * self.dtype.itemsize, ) else: + self._require_packed_device_rows("transfer_kv_all_layer") transfer_kv_all_layer( src_k_layers=device_k_data_ptrs, dst_k_layers=self.k_data_ptrs, @@ -516,7 +528,23 @@ class MHATokenToKVPoolHost(HostKVCache): dst_v=self.v_buffer, page_size=self.page_size, ) + elif self.can_use_jit and not self.device_rows_packed: + # The per-layer host views of page_first rows sit + # `layout_dim` apart, which the all-layer kernel can step. + jit_transfer_hicache_all_layer( + page_size=self.page_size, + k_ptr_dst=self.k_data_ptrs, + v_ptr_dst=self.v_data_ptrs, + indices_dst=host_indices, + k_ptr_src=device_k_data_ptrs, + v_ptr_src=device_v_data_ptrs, + indices_src=device_indices, + kv_cache_dst_stride_bytes=self.layout_dim, + kv_cache_src_stride_bytes=self.device_row_stride_bytes, + element_size=self.element_dim * self.dtype.itemsize, + ) else: + self._require_packed_device_rows("transfer_kv_all_layer_lf_pf") transfer_kv_all_layer_lf_pf( src_k_layers=device_k_data_ptrs, dst_k=self.k_buffer, @@ -529,6 +557,7 @@ class MHATokenToKVPoolHost(HostKVCache): num_layers=self.layer_num, ) elif self.layout == "page_head": + self._require_packed_device_rows("transfer_kv_all_layer_lf_ph") transfer_kv_all_layer_lf_ph( src_k_layers=device_k_data_ptrs, dst_k=self.k_buffer, diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index b60325f8be..322db9ffb8 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -148,6 +148,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.packed_device_kv_buffers = [ buffer for pool in device_pools for buffer in pool.kv_buffer ] + self._init_device_row_stride( + buf + for pool in (self.device_pool, *self.mtp_draft_device_pools) + for buf in getattr(pool, "kv_buffer", None) or () + ) self._init_write_back_staging_buffers() def _init_dummy( @@ -402,7 +407,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.staging_token_capacity = 0 self.staging_buffer = None self.can_use_write_back_jit = False - if self.layout != "page_first" or (_is_npu or _is_xpu or _is_mps): + # The staged kernel reads whole device pages, so it needs packed rows. + if ( + self.layout != "page_first" + or not self.device_rows_packed + or (_is_npu or _is_xpu or _is_mps) + ): return # The staged write-back JIT kernel builds with hipcc and has a ROCm @@ -680,6 +690,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): element_dim=self.kv_cache_dim, ) else: + self._require_packed_device_rows("transfer_kv_per_layer_mla") transfer_kv_per_layer_mla( src=self.kv_buffer[host_layer_id], dst=device_pool.kv_buffer[device_layer_id], @@ -698,6 +709,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): element_dim=self.kv_cache_dim, ) else: + self._require_packed_device_rows("transfer_kv_per_layer_mla_pf_lf") transfer_kv_per_layer_mla_pf_lf( src=self.kv_buffer, dst=device_pool.kv_buffer[device_layer_id], @@ -808,6 +820,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): element_dim=self.kv_cache_dim, ) else: + self._require_packed_device_rows("transfer_kv_per_layer_mla") transfer_kv_per_layer_mla( src=device_pool.kv_buffer[device_layer_id], dst=self.kv_buffer[host_layer_id], @@ -907,10 +920,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ptr_src=device_data_ptrs, indices_src=device_indices, cache_dst_stride_bytes=self.token_stride_size, - cache_src_stride_bytes=self.token_stride_size, + cache_src_stride_bytes=self.device_row_stride_bytes, element_size=self.kv_cache_dim * self.dtype.itemsize, ) else: + self._require_packed_device_rows("transfer_kv_all_layer_mla") transfer_kv_all_layer_mla( src_layers=device_data_ptrs, dst_layers=self.data_ptrs, @@ -929,7 +943,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): dst=self.kv_buffer, page_size=self.page_size, ) + elif self.can_use_jit and not self.device_rows_packed: + # The per-layer host views of page_first rows sit + # `layout_dim` apart, which the all-layer kernel can step. + jit_transfer_hicache_all_layer_mla( + page_size=self.page_size, + ptr_dst=self.data_ptrs, + indices_dst=host_indices, + ptr_src=device_data_ptrs, + indices_src=device_indices, + cache_dst_stride_bytes=self.layout_dim, + cache_src_stride_bytes=self.device_row_stride_bytes, + element_size=self.kv_cache_dim * self.dtype.itemsize, + ) else: + self._require_packed_device_rows("transfer_kv_all_layer_mla_lf_pf") transfer_kv_all_layer_mla_lf_pf( src_layers=device_data_ptrs, dst=self.kv_buffer, diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index 5659b3efc3..bf2914fa85 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -149,6 +149,34 @@ def _rel_proj_kernel_eligible(r: torch.Tensor) -> bool: ) +# Slot-stride alignment (elements) of the fused prologue store: kVecElems for +# bf16, kMXFP8Block for MXFP8. +_FUSED_KV_STORE_SLOT_ALIGN = {torch.bfloat16: 8, torch.float8_e4m3fn: 32} + + +def _fused_kv_store_eligible( + k_buf: torch.Tensor, v_buf: torch.Tensor, head_dim: int, dtype: torch.dtype +) -> bool: + """Whether the fused prologue can store K/V of `dtype` straight into these + pool buffers. + + It writes one NHD ``[slot, head, head_dim]`` row per loc, stepping slots by + the buffers' shared ``stride(0)``, so a strided per-layer view qualifies as + long as each row is contiguous. Other layouts (HND / vectorized_5d) keep the + backend store, which owns their quant and layout; conv, windows and qk-norm + stay fused either way. + """ + slot_stride = k_buf.stride(0) + return slot_stride % _FUSED_KV_STORE_SLOT_ALIGN[dtype] == 0 and all( + buf.dtype == dtype + and buf.dim() == 3 + and buf.shape[-1] == head_dim + and buf.stride()[1:] == (head_dim, 1) + and buf.stride(0) == slot_stride + for buf in (k_buf, v_buf) + ) + + class RelLogitsProj(nn.Module): def __init__(self, d_rel: int, rel_extent: int, *, deterministic: bool = False): super().__init__() @@ -438,16 +466,8 @@ class InklingAttention(nn.Module): pool = get_token_to_kv_pool() k_buf = pool.get_key_buffer(self.layer_id) v_buf = pool.get_value_buffer(self.layer_id) - # The fused store writes raw bf16 into an NHD [slot, head, head_dim] - # buffer indexed by loc. Take it only for that exact layout: FP8/MXFP8 - # (non-bf16) and HND/vectorized_5d (4D/5D, paged (page, head) index) - # pools keep the backend store, which owns their quant + layout. conv + - # windows + qk-norm stay fused regardless. - do_bf16_store = ( - k_buf.dtype == torch.bfloat16 - and k_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() + do_bf16_store = _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.bfloat16 ) sfk = sfv = None do_mxfp8_store = False @@ -456,14 +476,9 @@ class InklingAttention(nn.Module): ): sfk, sfv = pool.get_kv_scale_buffer(self.layer_id) do_mxfp8_store = ( - k_buf.dtype == torch.float8_e4m3fn - and v_buf.dtype == torch.float8_e4m3fn - and k_buf.dim() == 3 - and v_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and v_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() - and v_buf.is_contiguous() + _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.float8_e4m3fn + ) and sfk.dim() == 5 and sfv.dim() == 5 and getattr(pool, "page_size", 0) == 128 @@ -553,15 +568,8 @@ class InklingAttention(nn.Module): pool = get_token_to_kv_pool() k_buf = pool.get_key_buffer(self.layer_id) v_buf = pool.get_value_buffer(self.layer_id) - do_bf16_store = ( - k_buf.dtype == torch.bfloat16 - and v_buf.dtype == torch.bfloat16 - and k_buf.dim() == 3 - and v_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and v_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() - and v_buf.is_contiguous() + do_bf16_store = _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.bfloat16 ) sfk = sfv = None do_mxfp8_store = False @@ -570,14 +578,9 @@ class InklingAttention(nn.Module): ): sfk, sfv = pool.get_kv_scale_buffer(self.layer_id) do_mxfp8_store = ( - k_buf.dtype == torch.float8_e4m3fn - and v_buf.dtype == torch.float8_e4m3fn - and k_buf.dim() == 3 - and v_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and v_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() - and v_buf.is_contiguous() + _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.float8_e4m3fn + ) and sfk.dim() == 5 and sfv.dim() == 5 and getattr(pool, "page_size", 0) == 128 @@ -672,15 +675,8 @@ class InklingAttention(nn.Module): pool = get_token_to_kv_pool() k_buf = pool.get_key_buffer(self.layer_id) v_buf = pool.get_value_buffer(self.layer_id) - do_bf16_store = ( - k_buf.dtype == torch.bfloat16 - and v_buf.dtype == torch.bfloat16 - and k_buf.dim() == 3 - and v_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and v_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() - and v_buf.is_contiguous() + do_bf16_store = _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.bfloat16 ) sfk = sfv = None do_mxfp8_store = False @@ -689,14 +685,9 @@ class InklingAttention(nn.Module): ): sfk, sfv = pool.get_kv_scale_buffer(self.layer_id) do_mxfp8_store = ( - k_buf.dtype == torch.float8_e4m3fn - and v_buf.dtype == torch.float8_e4m3fn - and k_buf.dim() == 3 - and v_buf.dim() == 3 - and k_buf.shape[-1] == self.head_dim - and v_buf.shape[-1] == self.head_dim - and k_buf.is_contiguous() - and v_buf.is_contiguous() + _fused_kv_store_eligible( + k_buf, v_buf, self.head_dim, torch.float8_e4m3fn + ) and sfk.dim() == 5 and sfv.dim() == 5 and getattr(pool, "page_size", 0) == 128 diff --git a/python/sglang/test/kernels/kv_canary/_fixtures.py b/python/sglang/test/kernels/kv_canary/_fixtures.py index c3cd5bdb15..10d7b4bf9c 100644 --- a/python/sglang/test/kernels/kv_canary/_fixtures.py +++ b/python/sglang/test/kernels/kv_canary/_fixtures.py @@ -83,6 +83,7 @@ def make_real_kv_source( page_size: int = 1, read_bytes: Optional[int] = None, pad_dim1: int = 0, + row_gap: int = 0, device: torch.device, fill: int = 0, ) -> RealKvSource: @@ -90,12 +91,18 @@ def make_real_kv_source( ``pad_dim1`` adds trailing per-row bytes the canary should skip — used by the "holey dim 1" case to confirm the kernel never reads past ``page_size * num_bytes_per_token``. + + ``row_gap`` makes the source a strided view: each row is followed by ``row_gap`` bytes that belong + to the backing buffer but not to the view (filled with the complement of ``fill``), the way a + per-layer view into a larger KV entry is laid out. The kernel must step rows by ``stride(0)``. """ num_rows = (num_slots + page_size - 1) // page_size cols = page_size * num_bytes_per_token + pad_dim1 - tensor = torch.full( - (num_rows, cols), fill_value=fill, dtype=torch.uint8, device=device + backing = torch.full( + (num_rows, cols + row_gap), fill_value=fill, dtype=torch.uint8, device=device ) + backing[:, cols:] = ~fill & 0xFF + tensor = backing[:, :cols] effective_read = read_bytes if read_bytes is not None else num_bytes_per_token return RealKvSource( tensor=tensor, @@ -117,6 +124,7 @@ def make_real_kv_sources( device: torch.device, rng: Optional[random.Random] = None, fill_strategy: FillStrategy = "constant_per_source", + row_gap: int = 0, ) -> tuple[RealKvSource, ...]: sources: list[RealKvSource] = [] for i in range(count): @@ -126,6 +134,7 @@ def make_real_kv_sources( num_bytes_per_token=num_bytes_per_token, page_size=page_size, read_bytes=read_bytes_eff, + row_gap=row_gap, device=device, fill=(i + 1) * 17, ) diff --git a/test/registered/kernels/ops/kv_canary/test_verify_hand.py b/test/registered/kernels/ops/kv_canary/test_verify_hand.py index a71c53ac6d..8bc6a2ef72 100644 --- a/test/registered/kernels/ops/kv_canary/test_verify_hand.py +++ b/test/registered/kernels/ops/kv_canary/test_verify_hand.py @@ -1132,6 +1132,47 @@ class TestRealKvSource: assert _n_violations(cuda_log) == 0 + def test_real_kv_source_strided_rows(self) -> None: + """A strided source (``stride(0) > shape[1]``) is folded row by ``stride(0)``. + + The gap after each row is filled with bytes the view does not own; stepping rows by + ``shape[1]`` would fold them into the hash and break the clean chain. + """ + buf_pair = _buf_pair() + sources = make_real_kv_sources( + count=1, + num_slots=16, + row_gap=32, + device=_DEVICE, + fill_strategy="random_bytes", + ) + assert sources[0].tensor.stride(0) > sources[0].tensor.shape[1] + sources_ref = clone_real_kv_sources(sources) + + _stamp_clean_kv_chain( + buf_pair=buf_pair, + sources_cuda=sources, + input_ids=torch.tensor([1, 2, 3], dtype=torch.int64, device=_DEVICE), + positions=torch.tensor([0, 1, 2], dtype=torch.int64, device=_DEVICE), + out_cache_loc=torch.tensor([1, 2, 5], dtype=torch.int64, device=_DEVICE), + real_kv_hash_mode=consts.RealKvHashMode.ALL, + ) + + plan_pair = make_verify_plan_pair( + slot_indices=[1, 2, 5], + positions=[0, 1, 2], + prev_slot_indices=[-1, 1, 2], + device=_DEVICE, + ) + cuda_log, _ = run_verify_diff( + buf_pair=buf_pair, + plan_pair=plan_pair, + real_kv_sources_pair=(sources, sources_ref), + real_kv_hash_mode=consts.RealKvHashMode.ALL, + ) + + assert _n_violations(cuda_log) == 0 + class TestLayoutAndScheduling: def test_page_size_gt_1_access_pattern(self) -> None: diff --git a/test/registered/kernels/ops/kv_canary/test_write_hand.py b/test/registered/kernels/ops/kv_canary/test_write_hand.py index 79ba9f2602..fb49549756 100644 --- a/test/registered/kernels/ops/kv_canary/test_write_hand.py +++ b/test/registered/kernels/ops/kv_canary/test_write_hand.py @@ -796,6 +796,33 @@ class TestRealKvHash: real_kv_hash_mode=mode, ) + @pytest.mark.parametrize("page_size", [1, 4]) + def test_strided_real_kv_source_byte_equal(self, page_size: int) -> None: + """A strided source (``stride(0) > shape[1]``) folds the same hash as its packed copy. + + The reference folds a contiguous clone; the CUDA kernel folds the strided view, whose + rows are followed by bytes it does not own. They agree only if the kernel steps rows + by ``stride(0)``. + """ + sources_cuda = make_real_kv_sources( + count=2, + page_size=page_size, + row_gap=48, + device=_DEVICE, + fill_strategy="random_bytes", + ) + assert all(s.tensor.stride(0) > s.tensor.shape[1] for s in sources_cuda) + sources_ref = clone_real_kv_sources(sources_cuda) + + _run_write( + buf_pair=self.buf_pair, + input_ids=[10, 20, 30], + positions=[0, 1, 2], + out_cache_loc=[1, 6, 11], + real_kv_sources_pair=(sources_cuda, sources_ref), + real_kv_hash_mode=consts.RealKvHashMode.ALL, + ) + @pytest.mark.parametrize("count", [1, 2, 3, 4]) def test_real_kv_sources_fold_1_to_4(self, count: int) -> None: """Folding ``count`` sources sequentially → CUDA matches ref for every count in {1..4}.""" diff --git a/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py b/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py index c4e134a791..dbfcc8fa15 100644 --- a/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py +++ b/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py @@ -85,6 +85,49 @@ def test_fused_fp8_qkv_kv_cache( ) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize( + "hkv,head_dim,extra", [(8, 128, 1024), (2, 64, 128), (1, 128, 32)] +) +@pytest.mark.parametrize("num_tokens", [1, 7, 64]) +def test_fused_fp8_qkv_kv_cache_strided_cache(num_tokens, dtype, hkv, head_dim, extra): + """Caches that are strided views (slot stride > kv_dim), like the unified + pool's per-layer views: rows land at ``slot * stride(0)`` and bytes outside + the view are never touched.""" + torch.manual_seed(0) + device = "cuda" + kv_dim = hkv * head_dim + total_slots = num_tokens + 4 + slot_stride = kv_dim + extra # fp8: elements == bytes + backing_k = torch.zeros(total_slots, slot_stride, dtype=FP8, device=device) + backing_v = torch.zeros_like(backing_k) + k_cache = backing_k[:, :kv_dim].view(total_slots, hkv, head_dim) + v_cache = backing_v[:, extra:].view(total_slots, hkv, head_dim) + assert not k_cache.is_contiguous() and k_cache.stride(0) == slot_stride + + k = torch.randn(num_tokens, hkv, head_dim, dtype=dtype, device=device) + v = torch.randn(num_tokens, hkv, head_dim, dtype=dtype, device=device) + cache_loc = torch.randperm(total_slots, device=device)[:num_tokens] + + assert fused_fp8_qkv_kv_cache(None, k, v, k_cache, v_cache, cache_loc) is None + + loc = cache_loc.long() + k_ref = _ref_quant(k.reshape(num_tokens, kv_dim).float(), 1.0) + v_ref = _ref_quant(v.reshape(num_tokens, kv_dim).float(), 1.0) + torch.testing.assert_close( + _bytes(k_cache.reshape(total_slots, kv_dim)[loc]), _bytes(k_ref), rtol=0, atol=0 + ) + torch.testing.assert_close( + _bytes(v_cache.reshape(total_slots, kv_dim)[loc]), _bytes(v_ref), rtol=0, atol=0 + ) + untouched = torch.ones(total_slots, dtype=torch.bool, device=device) + untouched[loc] = False + assert (backing_k[:, kv_dim:].view(torch.uint8) == 0).all() + assert (backing_v[:, :extra].view(torch.uint8) == 0).all() + assert (backing_k[untouched].view(torch.uint8) == 0).all() + assert (backing_v[untouched].view(torch.uint8) == 0).all() + + if __name__ == "__main__": import sys diff --git a/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py new file mode 100644 index 0000000000..c9f3b7476b --- /dev/null +++ b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py @@ -0,0 +1,95 @@ +"""HiCache host transfers address strided device KV rows by their stride. + +A per-layer device view can step its slots by more than one row: the unified +pool's token-major entries hold every layer's K and V in one slot. Backing +such rows up by the row width copies neighbouring bytes instead, and the +reload later restores them as if they were this token's KV -- silently. This +round-trips strided device rows through the host pool and compares them bit +for bit. + + python -m pytest test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py -v +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_L, _H, _D, _ROWS = 2, 2, 64, 64 +# One slot holds every layer's K and V plus a gap, as a unified-pool entry. +_E = 2 * _L * _H * _D + 128 + + +def _strided_device_pool(): + pool = MHATokenToKVPool( + size=_ROWS - 1, + page_size=1, + dtype=torch.bfloat16, + head_num=_H, + head_dim=_D, + layer_num=_L, + device="cuda", + enable_memory_saver=False, + ) + backing = torch.randn(_ROWS * _E, dtype=torch.bfloat16, device="cuda") + pool.k_buffer = [ + backing.as_strided((_ROWS, _H, _D), (_E, _D, 1), 2 * l * _H * _D) + for l in range(_L) + ] + pool.v_buffer = [ + backing.as_strided((_ROWS, _H, _D), (_E, _D, 1), (2 * l + 1) * _H * _D) + for l in range(_L) + ] + pool._init_data_ptrs_and_strides() + return pool + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestHiCacheStridedDeviceRows(unittest.TestCase): + def test_backup_and_load_keep_every_slot(self): + for layout in ("layer_first", "page_first"): + with self.subTest(layout=layout): + device = _strided_device_pool() + host = MHATokenToKVPoolHost( + device_pool=device, + host_to_device_ratio=2.0, + host_size=0, + page_size=1, + layout=layout, + pin_memory=True, + device="cpu", + allocator_type="default", + ) + if not host.can_use_jit: + self.skipTest("strided device rows need the JIT HiCache kernels") + + src = torch.tensor([3, 7, 20, 41], device="cuda") + dst = torch.tensor([5, 9, 30, 50], device="cuda") + want = [ + (k[src].clone(), v[src].clone()) + for k, v in zip(device.k_buffer, device.v_buffer) + ] + host_slots = host.alloc(len(src)).to("cuda") + + host.backup_from_device_all_layer(device, host_slots, src, "kernel") + for layer in range(_L): + host.load_to_device_per_layer( + device, host_slots, dst, layer, "kernel" + ) + torch.cuda.synchronize() + + for layer, (k, v) in enumerate(want): + self.assertTrue(torch.equal(device.k_buffer[layer][dst], k)) + self.assertTrue(torch.equal(device.v_buffer[layer][dst], v)) + # The staged write-back reads whole device pages: packed rows only. + self.assertEqual(host.device_row_stride_bytes, _E * 2) + self.assertFalse(host.can_use_write_back_jit) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/kvcache/test_masked_set_kv_buffer_strided.py b/test/registered/kernels/ops/kvcache/test_masked_set_kv_buffer_strided.py new file mode 100644 index 0000000000..9a4f17393d --- /dev/null +++ b/test/registered/kernels/ops/kvcache/test_masked_set_kv_buffer_strided.py @@ -0,0 +1,82 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""``masked_set_kv_buffer_kernel`` must step destination slots by the buffer's +own stride. The unified pool's per-layer K/V buffers are strided views (slot +stride > head_num * head_dim), so a hardcoded ``loc * H * D`` would write into +the wrong slot without any error. + + python -m pytest test/registered/kernels/ops/kvcache/test_masked_set_kv_buffer_strided.py -v +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import masked_set_kv_buffer_kernel +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestMaskedSetKvBufferStrided(unittest.TestCase): + @unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") + def test_writes_land_at_slot_stride(self): + torch.manual_seed(0) + N, H, D = 16, 4, 64 + slots = N + 8 + E = 2 * H * D + 128 # slot stride wider than one K row + device = "cuda" + backing_k = torch.zeros(slots * E, dtype=torch.bfloat16, device=device) + backing_v = torch.zeros_like(backing_k) + k_buffer = backing_k.as_strided((slots, H, D), (E, D, 1)) + v_buffer = backing_v.as_strided((slots, H, D), (E, D, 1), H * D) + cache_k = torch.randn(N, H, D, dtype=torch.bfloat16, device=device) + cache_v = torch.randn(N, H, D, dtype=torch.bfloat16, device=device) + loc = torch.randperm(slots, device=device)[:N] + mask = torch.arange(N, device=device) % 3 != 0 + + masked_set_kv_buffer_kernel[(N,)]( + cache_k, + cache_v, + k_buffer, + v_buffer, + loc, + mask, + N, + H, + D, + 128, + cache_k.stride(0), + cache_k.stride(1), + cache_v.stride(0), + cache_v.stride(1), + k_buffer.stride(0), + v_buffer.stride(0), + ) + + ref_k = torch.zeros_like(k_buffer) + ref_v = torch.zeros_like(v_buffer) + ref_k[loc[mask]] = cache_k[mask] + ref_v[loc[mask]] = cache_v[mask] + torch.testing.assert_close(k_buffer, ref_k, rtol=0, atol=0) + torch.testing.assert_close(v_buffer, ref_v, rtol=0, atol=0) + # Nothing outside the two views was touched. + gap_k = backing_k.view(slots, E)[:, H * D :] + gap_v = backing_v.view(slots, E)[:, : H * D] + self.assertTrue(bool((gap_k == 0).all()) and bool((gap_v == 0).all())) + self.assertTrue(bool((backing_v.view(slots, E)[:, 2 * H * D :] == 0).all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py index d6fb6de05f..dc7ad90f42 100644 --- a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py @@ -80,6 +80,34 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): get_mha_host_pool_cls(asymmetric_pool), AsymmetricMHATokenToKVPoolHost ) + def test_builds_over_a_real_device_pool(self): + # The asymmetric pool has per-side row widths and no `token_stride_size`, + # so building it must not need one for its packed device rows. + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + + device = MHATokenToKVPool( + size=15, + page_size=2, + dtype=torch.float16, + head_num=2, + head_dim=4, + v_head_dim=6, + layer_num=3, + device="cpu", + enable_memory_saver=False, + ) + host = AsymmetricMHATokenToKVPoolHost( + device_pool=device, + host_to_device_ratio=2.0, + host_size=0, + page_size=2, + layout="page_first", + pin_memory=False, + device="cpu", + allocator_type="default", + ) + self.assertTrue(host.device_rows_packed) + def test_staged_write_back_jit_uses_separate_kv_buffers(self): host = _make_host("page_first") host.page_num = 4 diff --git a/test/registered/unit/mem_cache/test_kv_stride_hygiene.py b/test/registered/unit/mem_cache/test_kv_stride_hygiene.py new file mode 100644 index 0000000000..ebca9db8b1 --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_stride_hygiene.py @@ -0,0 +1,171 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""KV-cache row addressing must come from tensor strides, not shapes. + +A per-layer KV buffer may be a strided view into a larger buffer (slot stride +larger than ``head_num * head_dim``). Every helper that computes a slot address +must then use ``stride(0)``; deriving it from ``prod(shape[1:])`` silently +addresses the wrong slot, and ``.contiguous()`` silently snapshots instead of +aliasing the live pool. A kernel that uses the slot stride as its copy width +(the tiled slot move) must refuse such views instead. These tests pin the +stride-derived behaviour on views that differ from contiguous buffers. + +CPU-only. + + python -m pytest test/registered/unit/mem_cache/test_kv_stride_hygiene.py -v +""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.utils import canonicalize_stride +from sglang.kernels.ops.kv_canary.verify import ( + RealKvSource, + _build_real_kv_source_abi, +) +from sglang.srt.kv_canary.pool_patcher.buffer_alloc import make_row_source +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _strided_rows(num_rows: int, row_elems: int, slot_elems: int, dtype, *, offset=0): + """A [num_rows, row_elems] view whose row stride is ``slot_elems`` (> row_elems).""" + backing = torch.zeros(num_rows * slot_elems + offset, dtype=dtype) + view = backing.as_strided((num_rows, row_elems), (slot_elems, 1), offset) + assert not view.is_contiguous() + return backing, view + + +class TestCanonicalizeStride(unittest.TestCase): + def test_degenerate_head_dim_is_fixed(self): + t = torch.empty(4 * 64 * 128).as_strided((4, 1, 64, 128), (8192, 128, 128, 1)) + out = canonicalize_stride(t) + self.assertEqual(tuple(out.stride()), (8192, 8192, 128, 1)) + self.assertEqual(out.data_ptr(), t.data_ptr()) + + def test_strided_view_without_degenerate_dims_is_untouched(self): + ps, H, D, E = 4, 8, 128, 8 * 128 + 512 # slot stride wider than a row + t = torch.empty(3 * ps * E).as_strided((3, ps, H, D), (ps * E, E, D, 1)) + self.assertIs(canonicalize_stride(t), t) + + def test_strided_view_keeps_non_degenerate_strides(self): + # Degenerate head dim on a strided view: only that stride is rewritten. + ps, D, E = 4, 128, 2048 + t = torch.empty(3 * ps * E).as_strided((3, 1, ps, D), (ps * E, E, E, 1)) + out = canonicalize_stride(t) + self.assertEqual(tuple(out.stride()), (ps * E, ps * E, E, 1)) + for p in range(3): + for s in range(ps): + self.assertEqual(out[p, 0, s].data_ptr(), t[p, 0, s].data_ptr()) + + def test_single_head_with_distinct_strides_is_untouched(self): + ps, D, E = 4, 128, 2048 + t = torch.empty(3 * ps * E).as_strided((3, ps, 1, D), (ps * E, E, D, 1)) + self.assertIs(canonicalize_stride(t), t) + + +class TestRealKvSourceStrides(unittest.TestCase): + def test_accepts_strided_rows(self): + _, view = _strided_rows(8, 64, 96, torch.uint8) + src = RealKvSource( + tensor=view, page_size=1, num_bytes_per_token=64, read_bytes=16 + ) + self.assertEqual(src.tensor.stride(0), 96) + + def test_rejects_unaligned_row_stride(self): + _, view = _strided_rows(8, 64, 72, torch.uint8) + with self.assertRaisesRegex(ValueError, "row stride must be a multiple of 16"): + RealKvSource( + tensor=view, page_size=1, num_bytes_per_token=64, read_bytes=16 + ) + + def test_rejects_row_narrower_than_page_despite_wide_stride(self): + # The 96-byte stride would fit 80 bytes, but only 64 belong to the row; + # the rest is the next layer's data in a strided pool. + _, view = _strided_rows(8, 64, 96, torch.uint8) + with self.assertRaisesRegex(ValueError, "row is 64 bytes wide"): + RealKvSource( + tensor=view, page_size=1, num_bytes_per_token=80, read_bytes=16 + ) + + def test_launch_abi_aliases_strided_rows(self): + _, view = _strided_rows(8, 64, 96, torch.uint8) + src = RealKvSource( + tensor=view, page_size=1, num_bytes_per_token=64, read_bytes=16 + ) + bufs, _ = _build_real_kv_source_abi(real_kv_sources=(src,), device="cpu") + self.assertEqual(bufs[0].data_ptr(), view.data_ptr()) + self.assertEqual(bufs[0].stride(0), 96) + + +class TestMakeRowSourceAliases(unittest.TestCase): + def test_strided_layer_view_is_aliased_not_copied(self): + N, H, D, E = 6, 2, 16, 2 * 16 + 32 # bf16 elems; slot stride wider than a row + backing = torch.zeros(N * E, dtype=torch.bfloat16) + layer = backing.as_strided((N, H, D), (E, D, 1)) + (src,) = make_row_source(layer_buffer=layer, read_bytes=16) + self.assertEqual(src.tensor.data_ptr(), layer.data_ptr()) + self.assertEqual(tuple(src.tensor.shape), (N, H * D * 2)) + self.assertEqual(src.tensor.stride(0), E * 2) + layer[3, 1, 0] = 1.0 # a later pool write must be visible to the canary + written = src.tensor[3, D * 2 : D * 2 + 2].view(torch.bfloat16)[0] + self.assertNotEqual(int(written), 0) + + def test_non_viewable_layout_raises_instead_of_snapshotting(self): + layer = torch.zeros(4, 16, 2, dtype=torch.bfloat16).transpose(1, 2) + with self.assertRaises(RuntimeError): + make_row_source(layer_buffer=layer, read_bytes=16) + + +def _strided_mha_pool(L=2, N=5, H=2, D=16, E=2 * (2 * 16) + 64): + """An MHA pool whose K and V of every layer share one slot of E elements.""" + backing = torch.zeros(N * E, dtype=torch.float16) + pool = MHATokenToKVPool.__new__(MHATokenToKVPool) + pool.device = "cpu" + pool.layer_num = L + pool.k_buffer = [ + backing.as_strided((N, H, D), (E, D, 1), l * 2 * H * D) for l in range(L) + ] + pool.v_buffer = [ + backing.as_strided((N, H, D), (E, D, 1), (l * 2 + 1) * H * D) for l in range(L) + ] + return pool + + +class TestDataStridesFollowViews(unittest.TestCase): + def test_data_strides_use_stride0(self): + L, H, D, E = 2, 2, 16, 2 * (2 * 16) + 64 + itemsize = 2 + pool = _strided_mha_pool(L=L, H=H, D=D, E=E) + pool._init_data_ptrs_and_strides() + self.assertEqual(pool.data_strides.tolist(), [E * itemsize] * (2 * L)) + self.assertNotEqual(E * itemsize, H * D * itemsize) + self.assertEqual( + pool.data_ptrs.tolist(), + [x.data_ptr() for x in pool.k_buffer + pool.v_buffer], + ) + + def test_tiled_kv_copy_refuses_strided_views(self): + # One slot stride of bytes per slot is a whole entry on these views. + pool = _strided_mha_pool() + pool._init_data_ptrs_and_strides() + with self.assertRaisesRegex(AssertionError, "contiguous KV buffers"): + pool._init_kv_copy_and_warmup() + + +if __name__ == "__main__": + unittest.main()