[unified-memory] Derive KV row addresses from strides, not shapes (1/7) (#40326)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:07:47 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 5423a4d885
commit d3dbb71e5a
21 changed files with 733 additions and 118 deletions
@@ -24,6 +24,9 @@ struct FusedQkvParams {
int64_t q_stride;
int64_t k_stride;
int64_t v_stride;
// Elements between consecutive cache slots; may exceed kv_dim (strided per-layer views).
int64_t k_cache_stride;
int64_t v_cache_stride;
uint32_t num_tokens;
uint32_t q_dim;
uint32_t kv_dim;
@@ -76,12 +79,12 @@ __global__ void fused_fp8_qkv_kv_cache_kernel(const __grid_constant__ FusedQkvPa
}
quant_row<T, kVecN>(
static_cast<const T*>(params.k) + static_cast<size_t>(token) * params.k_stride,
static_cast<fp8_e4m3_t*>(params.k_cache) + static_cast<size_t>(slot) * params.kv_dim,
static_cast<fp8_e4m3_t*>(params.k_cache) + static_cast<size_t>(slot) * params.k_cache_stride,
params.kv_dim,
inv_k);
quant_row<T, kVecN>(
static_cast<const T*>(params.v) + static_cast<size_t>(token) * params.v_stride,
static_cast<fp8_e4m3_t*>(params.v_cache) + static_cast<size_t>(slot) * params.kv_dim,
static_cast<fp8_e4m3_t*>(params.v_cache) + static_cast<size_t>(slot) * params.v_cache_stride,
params.kv_dim,
inv_v);
@@ -127,13 +130,16 @@ struct FusedFp8QkvKvCache {
auto S = SymbolicSize{"num_slots"};
auto SK = SymbolicSize{"k_stride"};
auto SV = SymbolicSize{"v_stride"};
auto SKC = SymbolicSize{"k_cache_stride"};
auto SVC = SymbolicSize{"v_cache_stride"};
auto device = SymbolicDevice{};
auto idx_dtype = SymbolicDType{};
device.set_options<kDLCUDA>();
TensorMatcher({N, Dkv}).with_strides({SK, 1}).with_dtype<T>().with_device(device).verify(k);
TensorMatcher({N, Dkv}).with_strides({SV, 1}).with_dtype<T>().with_device(device).verify(v);
TensorMatcher({S, Dkv}).with_dtype<fp8_e4m3_t>().with_device(device).verify(k_cache).verify(v_cache);
TensorMatcher({S, Dkv}).with_strides({SKC, 1}).with_dtype<fp8_e4m3_t>().with_device(device).verify(k_cache);
TensorMatcher({S, Dkv}).with_strides({SVC, 1}).with_dtype<fp8_e4m3_t>().with_device(device).verify(v_cache);
TensorMatcher({N}).with_dtype<int32_t, int64_t>(idx_dtype).with_device(device).verify(cache_loc);
TensorMatcher({1}).with_dtype<fp32_t>().with_device(device).verify(k_scale).verify(v_scale);
@@ -156,12 +162,17 @@ struct FusedFp8QkvKvCache {
const uint32_t kv_dim = static_cast<uint32_t>(Dkv.unwrap());
const int64_t k_stride = SK.unwrap();
const int64_t v_stride = SV.unwrap();
const int64_t k_cache_stride = SKC.unwrap();
const int64_t v_cache_stride = SVC.unwrap();
RuntimeCheck(num_tokens > 0, "fused_fp8_qkv_kv_cache: num_tokens must be > 0, got ", num_tokens);
auto fits = [&](int vec) {
const int in_bytes = vec * static_cast<int>(sizeof(T));
const int out_bytes = vec * static_cast<int>(sizeof(fp8_e4m3_t));
bool ok = kv_dim % vec == 0 && k_stride % vec == 0 && v_stride % vec == 0 && aligned(k.data_ptr(), in_bytes) &&
aligned(v.data_ptr(), in_bytes);
ok = ok && k_cache_stride % vec == 0 && v_cache_stride % vec == 0 && aligned(k_cache.data_ptr(), out_bytes) &&
aligned(v_cache.data_ptr(), out_bytes);
if (quantize_q) {
ok = ok && q_dim % vec == 0 && q_stride % vec == 0 && aligned(q_ptr, in_bytes);
}
@@ -182,6 +193,8 @@ struct FusedFp8QkvKvCache {
.q_stride = q_stride,
.k_stride = k_stride,
.v_stride = v_stride,
.k_cache_stride = k_cache_stride,
.v_cache_stride = v_cache_stride,
.num_tokens = num_tokens,
.q_dim = q_dim,
.kv_dim = kv_dim,
@@ -132,7 +132,7 @@ struct AttnPrologueParams {
int64_t inter_stride_b;
int64_t inter_stride_t;
int64_t inter_stride_w;
int64_t kv_buf_stride; // elems per KV slot row (= Hkv * head_dim)
int64_t kv_buf_stride; // elems between consecutive KV slots (>= Hkv * head_dim)
uint32_t T;
uint32_t q; // draft_token_num
uint32_t dq;
@@ -12,7 +12,7 @@ namespace canary {
// Device-side handle for one real-KV source.
struct RealKvSourceHandle {
const uint8_t* tensor; // raw uint8 byte pointer to the source tensor
int32_t row_stride_bytes; // tensor.shape[1] in bytes (may exceed page_size * num_bytes_per_token)
int32_t row_stride_bytes; // tensor.stride(0) in bytes (may exceed page_size * num_bytes_per_token)
int32_t page_size;
int32_t num_bytes_per_token;
int32_t read_bytes;
@@ -37,8 +37,8 @@ SGL_DEVICE uint64_t splitmix64_mix3(uint64_t a, uint64_t b, uint64_t c) {
// tensor[slot_idx // page_size,
// (slot_idx % page_size) * num_bytes_per_token + byte_offset]
//
// row_stride_bytes is the dim-1 size of the underlying tensor in bytes (which may exceed
// page_size * num_bytes_per_token; trailing bytes are skipped).
// row_stride_bytes is the dim-0 stride of the underlying tensor in bytes (which may exceed
// page_size * num_bytes_per_token; the remaining bytes are skipped).
SGL_DEVICE void real_kv_load_uint4(
const RealKvSourceHandle& src, int64_t slot_idx, int64_t byte_offset, uint64_t& word_lo, uint64_t& word_hi) {
const int64_t row = slot_idx / src.page_size;
@@ -205,24 +205,28 @@ struct CanaryVerifyKernel {
SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"};
SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"};
TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_0);
SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"};
SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"};
TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_1);
SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"};
SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"};
TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_2);
SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"};
SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"};
TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_3);
@@ -271,7 +275,7 @@ struct CanaryVerifyKernel {
tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3};
for (int s = 0; s < kMaxRealKvSources; ++s) {
p.sources[s].tensor = static_cast<const uint8_t*>(source_bufs[s].data_ptr());
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].size(1));
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].stride(0));
p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize];
p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken];
p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes];
@@ -251,24 +251,28 @@ inline void canary_write_step_cuda(
SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"};
SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"};
TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_0);
SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"};
SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"};
TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_1);
SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"};
SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"};
TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_2);
SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"};
SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"};
TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3})
.with_strides({-1, 1})
.with_dtype<uint8_t>()
.with_device<kDLGPU>(device_)
.verify(real_kv_buf_3);
@@ -329,7 +333,7 @@ inline void canary_write_step_cuda(
tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3};
for (int s = 0; s < kMaxRealKvSources; ++s) {
p.sources[s].tensor = static_cast<const uint8_t*>(source_bufs[s].data_ptr());
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].size(1));
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].stride(0));
p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize];
p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken];
p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes];
+12 -21
View File
@@ -70,30 +70,21 @@ if _is_cuda:
#
# See: https://github.com/flashinfer-ai/flashinfer/issues/2232
def canonicalize_stride(tensor: torch.Tensor) -> torch.Tensor:
"""
Adjust degenerate strides for a tensor, make it canonical.
"""Give each size-1 dim whose stride collides with the next dim's a
canonical stride; every other stride is kept, so a non-contiguous KV view
keeps its addressing.
Example: shape ``[num_pages, 1, 64, 128]`` with stride
``[8192, 128, 128, 1]`` becomes stride ``[8192, 8192, 128, 1]``.
"""
sizes = tensor.size()
strides = tensor.stride()
ndim = tensor.dim()
need_fix = any(
sizes[i] == 1 and strides[i] == strides[i + 1] for i in range(ndim - 1)
)
if not need_fix:
strides = list(tensor.stride())
new_strides = list(strides)
for i in range(tensor.dim() - 2, -1, -1):
if sizes[i] == 1 and strides[i] == strides[i + 1]:
new_strides[i] = new_strides[i + 1] * sizes[i + 1]
if new_strides == strides:
return tensor
# canonicalize the stride
# Example:
# - shape: [num_pages, 1, 64, 128]
# - stride: [8192, 128, 128, 1] (wrong!)
# Gives new stride: [8192, 8192, 128 ,1] (correct!)
new_strides = [0] * ndim
new_strides[-1] = 1
for i in range(ndim - 2, -1, -1):
new_strides[i] = new_strides[i + 1] * sizes[i + 1]
return tensor.as_strided(sizes, new_strides)
+19 -11
View File
@@ -56,22 +56,23 @@ class RealKvSource:
num_bytes_per_token``. Trailing bytes of each row are ignored by the canary; this is exactly how the
abstraction accommodates pools whose per-row layout interleaves canary-relevant bytes with other metadata
(layer-split storage, K/V interleaving, ...). When ``page_size == 1`` the pattern
collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case.
collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case. Rows are stepped by
``tensor.stride(0)``, so dim 0 may be a per-layer view into a larger buffer.
A pool may expose multiple RealKvSource instances per (canary buffer × K/V half) — the launch wrappers
iterate the source list and fold each into the running real_kv_hash via splitmix64 (one int64 fingerprint
per slot, regardless of source count).
Pool patchers construct sources by:
- viewing / reshaping the underlying KV layer into the canonical [num_rows, dim1_bytes] form (no stage-copy
needed when the underlying storage is already row-major contiguous on dim 0),
- viewing the underlying KV layer as [num_rows, dim1_bytes] (a view, never a stage-copy: a copy would
fingerprint a stale snapshot instead of the live pool),
- choosing ``page_size`` and ``num_bytes_per_token`` so that the access pattern above lands on the bytes
the canary should fingerprint,
- leaving any per-row padding / non-canary bytes in the trailing portion of each row (they will simply be
skipped).
16-byte alignment precondition: the CUDA fold kernel issues 128-bit aligned loads, so ``read_bytes``,
``num_bytes_per_token``, and the row stride (``tensor.shape[1]`` in bytes) must all be positive
``num_bytes_per_token``, and the row stride (``tensor.stride(0)`` in bytes) must all be positive
multiples of 16. There is no "skip this source" sentinel — callers omit the source from their
``real_kv_sources`` tuple entirely (factory helpers return an empty tuple in that case).
@@ -114,19 +115,21 @@ class RealKvSource:
raise ValueError(
f"kv-canary: RealKvSource.tensor must be at least 2-D, got shape {tuple(self.tensor.shape)}"
)
row_stride_bytes = int(self.tensor.shape[1]) * self.tensor.element_size()
row_stride_bytes = int(self.tensor.stride(0)) * self.tensor.element_size()
if row_stride_bytes % 16 != 0:
raise ValueError(
f"kv-canary: RealKvSource.tensor dim-1 byte width must be a multiple of 16, "
f"kv-canary: RealKvSource.tensor row stride must be a multiple of 16 bytes, "
f"got {row_stride_bytes} bytes (shape={tuple(self.tensor.shape)}, "
f"dtype={self.tensor.dtype})"
f"strides={tuple(self.tensor.stride())}, dtype={self.tensor.dtype})"
)
# A row is addressed as page_size slots of num_bytes_per_token, unchecked at fold time;
# a narrower row hashes fewer bytes than asked and still reports the chain clean.
# a narrower row hashes fewer bytes than asked and still reports the chain clean. Check the
# row's width, not its stride.
row_bytes = int(self.tensor.shape[1]) * self.tensor.element_size()
min_row_bytes = self.page_size * self.num_bytes_per_token
if row_stride_bytes < min_row_bytes:
if row_bytes < min_row_bytes:
raise ValueError(
f"kv-canary: RealKvSource.tensor dim-1 is {row_stride_bytes} bytes but "
f"kv-canary: RealKvSource.tensor row is {row_bytes} bytes wide but "
f"page_size={self.page_size} x num_bytes_per_token={self.num_bytes_per_token} "
f"needs {min_row_bytes} (shape={tuple(self.tensor.shape)}, "
f"dtype={self.tensor.dtype})"
@@ -412,7 +415,12 @@ def _build_real_kv_source_abi(
)
for i, source in enumerate(real_kv_sources):
_assert_contiguous(source.tensor, f"real_kv_sources[{i}].tensor")
# The kernels step rows by stride(0), so only the bytes within a row must be packed.
if source.tensor.stride(-1) != 1:
raise ValueError(
f"kv-canary: real_kv_sources[{i}].tensor rows must be packed "
f"(stride(-1) == 1), got strides {tuple(source.tensor.stride())}"
)
source_u8 = source.tensor.view(torch.uint8)
if source_u8.dim() != 2:
raise ValueError(
@@ -67,11 +67,10 @@ def make_row_source(
layer_buffer: torch.Tensor,
read_bytes: int,
) -> Tuple[RealKvSource, ...]:
contiguous = layer_buffer.contiguous()
num_slots = int(contiguous.shape[0])
num_slots = int(layer_buffer.shape[0])
if num_slots == 0 or read_bytes == 0:
return ()
flat = contiguous.view(torch.uint8).reshape(num_slots, -1)
flat = layer_buffer.view(torch.uint8).view(num_slots, -1)
num_bytes_per_token = int(flat.shape[1])
clipped = _clip_read_bytes_aligned(
requested=read_bytes, num_bytes_per_token=num_bytes_per_token
+20 -16
View File
@@ -2117,6 +2117,13 @@ class MHATokenToKVPool(KVCache):
self._kv_copy_config = None
return
# The tiled copy uses `data_strides[i]` as both slot stride and copy width.
for buf in self._slot_move_pointer_buffers():
assert buf.is_contiguous(), (
"the tiled KV copy needs contiguous KV buffers; got shape "
f"{tuple(buf.shape)}, strides {buf.stride()}"
)
# Heuristics for KV copy tiling
_KV_COPY_STRIDE_THRESHOLD_LARGE = 8192
_KV_COPY_STRIDE_THRESHOLD_MEDIUM = 4096
@@ -2292,10 +2299,7 @@ class MHATokenToKVPool(KVCache):
device=self.device,
)
self.data_strides = torch.tensor(
[
np.prod(x.shape[1:]) * x.dtype.itemsize
for x in slot_move_pointer_buffers
],
[x.stride(0) * x.dtype.itemsize for x in slot_move_pointer_buffers],
device=self.device,
)
@@ -2647,11 +2651,13 @@ class MHATokenToKVPool(KVCache):
if dcp_kv_mask is not None:
N, H, D = cache_k.shape
k_buf = self.k_buffer[layer_id - self.start_layer]
v_buf = self.v_buffer[layer_id - self.start_layer]
masked_set_kv_buffer_kernel[(N,)](
cache_k,
cache_v,
self.k_buffer[layer_id - self.start_layer],
self.v_buffer[layer_id - self.start_layer],
k_buf,
v_buf,
loc,
dcp_kv_mask,
N,
@@ -2662,6 +2668,8 @@ class MHATokenToKVPool(KVCache):
cache_k.stride(1),
cache_v.stride(0),
cache_v.stride(1),
k_buf.stride(0),
v_buf.stride(0),
)
return
@@ -3224,10 +3232,7 @@ class NoOpMHATokenToKVPool(MHATokenToKVPool):
)
self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0)
self.data_strides = torch.tensor(
[
np.prod(x.shape[1:]) * x.dtype.itemsize
for x in self.k_buffer + self.v_buffer
],
[x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer],
device=self.device,
)
@@ -3610,10 +3615,7 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
)
self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0)
self.data_strides = torch.tensor(
[
np.prod(x.shape[1:]) * x.dtype.itemsize
for x in self.k_buffer + self.v_buffer
],
[x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer],
device=self.device,
)
# This override replaces the base allocation, so the PD-transfer
@@ -5295,6 +5297,8 @@ def masked_set_kv_buffer_kernel(
k_stride_H: tl.constexpr,
v_stride_B: tl.constexpr,
v_stride_H: tl.constexpr,
k_buffer_stride: tl.constexpr,
v_buffer_stride: tl.constexpr,
):
pid = tl.program_id(0)
if pid >= N:
@@ -5316,10 +5320,10 @@ def masked_set_kv_buffer_kernel(
col = idx % D
key = tl.load(k_ptr + pid * k_stride_B + row * k_stride_H + col, mask=mask)
tl.store(k_buffer_ptr + loc * H * D + idx, key, mask=mask)
tl.store(k_buffer_ptr + loc * k_buffer_stride + idx, key, mask=mask)
value = tl.load(v_ptr + pid * v_stride_B + row * v_stride_H + col, mask=mask)
tl.store(v_buffer_ptr + loc * H * D + idx, value, mask=mask)
tl.store(v_buffer_ptr + loc * v_buffer_stride + idx, value, mask=mask)
class MHATokenToKOnlyPool(KVCache):
@@ -278,6 +278,54 @@ class HostKVCache(abc.ABC):
_cuda_host_unregister(buf)
self.kv_buffer = None
# Bytes between strided device KV rows; None (the default, and what a pool
# built without `_init_device_row_stride` keeps) means packed rows.
_strided_device_row_bytes: Optional[int] = None
def _init_device_row_stride(self, device_buffers: Iterable[torch.Tensor]) -> None:
"""Record the row stride of strided device KV buffers.
A strided per-layer view -- the unified pool's token-major entries --
steps its rows by `stride(0)` rather than by the row width, and the
device-side addresses in the transfer kernels must follow it.
"""
buffers = list(device_buffers)
strides = {
buf.stride(0) * buf.element_size()
for buf in buffers
if not buf.is_contiguous()
}
if not strides:
return
if len(strides) > 1 or any(buf.is_contiguous() for buf in buffers):
raise NotImplementedError(
"HiCache: the device KV buffers do not share one row stride "
f"(strided rows {sorted(strides)} B, mixed with packed ones)"
)
for buf in buffers:
assert buf.stride(-1) == 1, (
f"HiCache needs packed device rows; got strides {buf.stride()}"
)
self._strided_device_row_bytes = strides.pop()
@property
def device_rows_packed(self) -> bool:
return self._strided_device_row_bytes is None
@property
def device_row_stride_bytes(self) -> int:
"""Bytes between consecutive device KV rows."""
return self._strided_device_row_bytes or self.token_stride_size
def _require_packed_device_rows(self, kernel: str) -> None:
if not self.device_rows_packed:
raise NotImplementedError(
f"HiCache: {kernel} steps device rows by the row width "
f"({self.token_stride_size} B), but this device pool's rows are "
f"{self.device_row_stride_bytes} B apart. Use the JIT HiCache "
"kernels with a layer_first or page_first host layout."
)
@abc.abstractmethod
def get_size_per_token(self):
raise NotImplementedError()
+31 -2
View File
@@ -152,6 +152,13 @@ class MHATokenToKVPoolHost(HostKVCache):
pool_label=pool_label,
)
self.element_dim = self.head_num * self.head_dim
self._init_device_row_stride(
buf
for pool in (self.device_pool, *self.mtp_draft_device_pools)
if pool is not None
for side in ("k_buffer", "v_buffer")
for buf in getattr(pool, side, None) or ()
)
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
# write-back kernel has a ROCm path, so enable them on HIP too. This
@@ -268,7 +275,8 @@ class MHATokenToKVPoolHost(HostKVCache):
self.staging_k_buffer = None
self.staging_v_buffer = None
self.can_use_write_back_jit = False
if self.layout != "page_first":
# The staged kernel reads whole device pages, so it needs packed rows.
if self.layout != "page_first" or not self.device_rows_packed:
return
page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK)
staging = prepare_mha_write_back_staging(
@@ -328,6 +336,7 @@ class MHATokenToKVPoolHost(HostKVCache):
element_dim=self.element_dim,
)
else:
self._require_packed_device_rows("transfer_kv_per_layer")
transfer_kv_per_layer(
src_k=self.k_buffer[host_layer_id],
dst_k=device_pool.k_buffer[device_layer_id],
@@ -353,6 +362,7 @@ class MHATokenToKVPoolHost(HostKVCache):
element_dim=self.element_dim,
)
else:
self._require_packed_device_rows("transfer_kv_per_layer_pf_lf")
transfer_kv_per_layer_pf_lf(
src_k=self.k_buffer,
dst_k=device_pool.k_buffer[device_layer_id],
@@ -365,6 +375,7 @@ class MHATokenToKVPoolHost(HostKVCache):
src_layout_dim=self.layout_dim,
)
elif self.layout == "page_head":
self._require_packed_device_rows("transfer_kv_per_layer_ph_lf")
transfer_kv_per_layer_ph_lf(
src_k=self.k_buffer,
dst_k=device_pool.k_buffer[device_layer_id],
@@ -489,10 +500,11 @@ class MHATokenToKVPoolHost(HostKVCache):
v_ptr_src=device_v_data_ptrs,
indices_src=device_indices,
kv_cache_dst_stride_bytes=self.token_stride_size,
kv_cache_src_stride_bytes=self.token_stride_size,
kv_cache_src_stride_bytes=self.device_row_stride_bytes,
element_size=self.element_dim * self.dtype.itemsize,
)
else:
self._require_packed_device_rows("transfer_kv_all_layer")
transfer_kv_all_layer(
src_k_layers=device_k_data_ptrs,
dst_k_layers=self.k_data_ptrs,
@@ -516,7 +528,23 @@ class MHATokenToKVPoolHost(HostKVCache):
dst_v=self.v_buffer,
page_size=self.page_size,
)
elif self.can_use_jit and not self.device_rows_packed:
# The per-layer host views of page_first rows sit
# `layout_dim` apart, which the all-layer kernel can step.
jit_transfer_hicache_all_layer(
page_size=self.page_size,
k_ptr_dst=self.k_data_ptrs,
v_ptr_dst=self.v_data_ptrs,
indices_dst=host_indices,
k_ptr_src=device_k_data_ptrs,
v_ptr_src=device_v_data_ptrs,
indices_src=device_indices,
kv_cache_dst_stride_bytes=self.layout_dim,
kv_cache_src_stride_bytes=self.device_row_stride_bytes,
element_size=self.element_dim * self.dtype.itemsize,
)
else:
self._require_packed_device_rows("transfer_kv_all_layer_lf_pf")
transfer_kv_all_layer_lf_pf(
src_k_layers=device_k_data_ptrs,
dst_k=self.k_buffer,
@@ -529,6 +557,7 @@ class MHATokenToKVPoolHost(HostKVCache):
num_layers=self.layer_num,
)
elif self.layout == "page_head":
self._require_packed_device_rows("transfer_kv_all_layer_lf_ph")
transfer_kv_all_layer_lf_ph(
src_k_layers=device_k_data_ptrs,
dst_k=self.k_buffer,
+30 -2
View File
@@ -148,6 +148,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.packed_device_kv_buffers = [
buffer for pool in device_pools for buffer in pool.kv_buffer
]
self._init_device_row_stride(
buf
for pool in (self.device_pool, *self.mtp_draft_device_pools)
for buf in getattr(pool, "kv_buffer", None) or ()
)
self._init_write_back_staging_buffers()
def _init_dummy(
@@ -402,7 +407,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.staging_token_capacity = 0
self.staging_buffer = None
self.can_use_write_back_jit = False
if self.layout != "page_first" or (_is_npu or _is_xpu or _is_mps):
# The staged kernel reads whole device pages, so it needs packed rows.
if (
self.layout != "page_first"
or not self.device_rows_packed
or (_is_npu or _is_xpu or _is_mps)
):
return
# The staged write-back JIT kernel builds with hipcc and has a ROCm
@@ -680,6 +690,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
element_dim=self.kv_cache_dim,
)
else:
self._require_packed_device_rows("transfer_kv_per_layer_mla")
transfer_kv_per_layer_mla(
src=self.kv_buffer[host_layer_id],
dst=device_pool.kv_buffer[device_layer_id],
@@ -698,6 +709,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
element_dim=self.kv_cache_dim,
)
else:
self._require_packed_device_rows("transfer_kv_per_layer_mla_pf_lf")
transfer_kv_per_layer_mla_pf_lf(
src=self.kv_buffer,
dst=device_pool.kv_buffer[device_layer_id],
@@ -808,6 +820,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
element_dim=self.kv_cache_dim,
)
else:
self._require_packed_device_rows("transfer_kv_per_layer_mla")
transfer_kv_per_layer_mla(
src=device_pool.kv_buffer[device_layer_id],
dst=self.kv_buffer[host_layer_id],
@@ -907,10 +920,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
ptr_src=device_data_ptrs,
indices_src=device_indices,
cache_dst_stride_bytes=self.token_stride_size,
cache_src_stride_bytes=self.token_stride_size,
cache_src_stride_bytes=self.device_row_stride_bytes,
element_size=self.kv_cache_dim * self.dtype.itemsize,
)
else:
self._require_packed_device_rows("transfer_kv_all_layer_mla")
transfer_kv_all_layer_mla(
src_layers=device_data_ptrs,
dst_layers=self.data_ptrs,
@@ -929,7 +943,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
dst=self.kv_buffer,
page_size=self.page_size,
)
elif self.can_use_jit and not self.device_rows_packed:
# The per-layer host views of page_first rows sit
# `layout_dim` apart, which the all-layer kernel can step.
jit_transfer_hicache_all_layer_mla(
page_size=self.page_size,
ptr_dst=self.data_ptrs,
indices_dst=host_indices,
ptr_src=device_data_ptrs,
indices_src=device_indices,
cache_dst_stride_bytes=self.layout_dim,
cache_src_stride_bytes=self.device_row_stride_bytes,
element_size=self.kv_cache_dim * self.dtype.itemsize,
)
else:
self._require_packed_device_rows("transfer_kv_all_layer_mla_lf_pf")
transfer_kv_all_layer_mla_lf_pf(
src_layers=device_data_ptrs,
dst=self.kv_buffer,
+43 -52
View File
@@ -149,6 +149,34 @@ def _rel_proj_kernel_eligible(r: torch.Tensor) -> bool:
)
# Slot-stride alignment (elements) of the fused prologue store: kVecElems for
# bf16, kMXFP8Block for MXFP8.
_FUSED_KV_STORE_SLOT_ALIGN = {torch.bfloat16: 8, torch.float8_e4m3fn: 32}
def _fused_kv_store_eligible(
k_buf: torch.Tensor, v_buf: torch.Tensor, head_dim: int, dtype: torch.dtype
) -> bool:
"""Whether the fused prologue can store K/V of `dtype` straight into these
pool buffers.
It writes one NHD ``[slot, head, head_dim]`` row per loc, stepping slots by
the buffers' shared ``stride(0)``, so a strided per-layer view qualifies as
long as each row is contiguous. Other layouts (HND / vectorized_5d) keep the
backend store, which owns their quant and layout; conv, windows and qk-norm
stay fused either way.
"""
slot_stride = k_buf.stride(0)
return slot_stride % _FUSED_KV_STORE_SLOT_ALIGN[dtype] == 0 and all(
buf.dtype == dtype
and buf.dim() == 3
and buf.shape[-1] == head_dim
and buf.stride()[1:] == (head_dim, 1)
and buf.stride(0) == slot_stride
for buf in (k_buf, v_buf)
)
class RelLogitsProj(nn.Module):
def __init__(self, d_rel: int, rel_extent: int, *, deterministic: bool = False):
super().__init__()
@@ -438,16 +466,8 @@ class InklingAttention(nn.Module):
pool = get_token_to_kv_pool()
k_buf = pool.get_key_buffer(self.layer_id)
v_buf = pool.get_value_buffer(self.layer_id)
# The fused store writes raw bf16 into an NHD [slot, head, head_dim]
# buffer indexed by loc. Take it only for that exact layout: FP8/MXFP8
# (non-bf16) and HND/vectorized_5d (4D/5D, paged (page, head) index)
# pools keep the backend store, which owns their quant + layout. conv +
# windows + qk-norm stay fused regardless.
do_bf16_store = (
k_buf.dtype == torch.bfloat16
and k_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
do_bf16_store = _fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.bfloat16
)
sfk = sfv = None
do_mxfp8_store = False
@@ -456,14 +476,9 @@ class InklingAttention(nn.Module):
):
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
do_mxfp8_store = (
k_buf.dtype == torch.float8_e4m3fn
and v_buf.dtype == torch.float8_e4m3fn
and k_buf.dim() == 3
and v_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and v_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
and v_buf.is_contiguous()
_fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
)
and sfk.dim() == 5
and sfv.dim() == 5
and getattr(pool, "page_size", 0) == 128
@@ -553,15 +568,8 @@ class InklingAttention(nn.Module):
pool = get_token_to_kv_pool()
k_buf = pool.get_key_buffer(self.layer_id)
v_buf = pool.get_value_buffer(self.layer_id)
do_bf16_store = (
k_buf.dtype == torch.bfloat16
and v_buf.dtype == torch.bfloat16
and k_buf.dim() == 3
and v_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and v_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
and v_buf.is_contiguous()
do_bf16_store = _fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.bfloat16
)
sfk = sfv = None
do_mxfp8_store = False
@@ -570,14 +578,9 @@ class InklingAttention(nn.Module):
):
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
do_mxfp8_store = (
k_buf.dtype == torch.float8_e4m3fn
and v_buf.dtype == torch.float8_e4m3fn
and k_buf.dim() == 3
and v_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and v_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
and v_buf.is_contiguous()
_fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
)
and sfk.dim() == 5
and sfv.dim() == 5
and getattr(pool, "page_size", 0) == 128
@@ -672,15 +675,8 @@ class InklingAttention(nn.Module):
pool = get_token_to_kv_pool()
k_buf = pool.get_key_buffer(self.layer_id)
v_buf = pool.get_value_buffer(self.layer_id)
do_bf16_store = (
k_buf.dtype == torch.bfloat16
and v_buf.dtype == torch.bfloat16
and k_buf.dim() == 3
and v_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and v_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
and v_buf.is_contiguous()
do_bf16_store = _fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.bfloat16
)
sfk = sfv = None
do_mxfp8_store = False
@@ -689,14 +685,9 @@ class InklingAttention(nn.Module):
):
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
do_mxfp8_store = (
k_buf.dtype == torch.float8_e4m3fn
and v_buf.dtype == torch.float8_e4m3fn
and k_buf.dim() == 3
and v_buf.dim() == 3
and k_buf.shape[-1] == self.head_dim
and v_buf.shape[-1] == self.head_dim
and k_buf.is_contiguous()
and v_buf.is_contiguous()
_fused_kv_store_eligible(
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
)
and sfk.dim() == 5
and sfv.dim() == 5
and getattr(pool, "page_size", 0) == 128
@@ -83,6 +83,7 @@ def make_real_kv_source(
page_size: int = 1,
read_bytes: Optional[int] = None,
pad_dim1: int = 0,
row_gap: int = 0,
device: torch.device,
fill: int = 0,
) -> RealKvSource:
@@ -90,12 +91,18 @@ def make_real_kv_source(
``pad_dim1`` adds trailing per-row bytes the canary should skip — used by the "holey dim 1" case to
confirm the kernel never reads past ``page_size * num_bytes_per_token``.
``row_gap`` makes the source a strided view: each row is followed by ``row_gap`` bytes that belong
to the backing buffer but not to the view (filled with the complement of ``fill``), the way a
per-layer view into a larger KV entry is laid out. The kernel must step rows by ``stride(0)``.
"""
num_rows = (num_slots + page_size - 1) // page_size
cols = page_size * num_bytes_per_token + pad_dim1
tensor = torch.full(
(num_rows, cols), fill_value=fill, dtype=torch.uint8, device=device
backing = torch.full(
(num_rows, cols + row_gap), fill_value=fill, dtype=torch.uint8, device=device
)
backing[:, cols:] = ~fill & 0xFF
tensor = backing[:, :cols]
effective_read = read_bytes if read_bytes is not None else num_bytes_per_token
return RealKvSource(
tensor=tensor,
@@ -117,6 +124,7 @@ def make_real_kv_sources(
device: torch.device,
rng: Optional[random.Random] = None,
fill_strategy: FillStrategy = "constant_per_source",
row_gap: int = 0,
) -> tuple[RealKvSource, ...]:
sources: list[RealKvSource] = []
for i in range(count):
@@ -126,6 +134,7 @@ def make_real_kv_sources(
num_bytes_per_token=num_bytes_per_token,
page_size=page_size,
read_bytes=read_bytes_eff,
row_gap=row_gap,
device=device,
fill=(i + 1) * 17,
)