[unified-memory] Derive KV row addresses from strides, not shapes (1/7) (#40326)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -24,6 +24,9 @@ struct FusedQkvParams {
|
||||
int64_t q_stride;
|
||||
int64_t k_stride;
|
||||
int64_t v_stride;
|
||||
// Elements between consecutive cache slots; may exceed kv_dim (strided per-layer views).
|
||||
int64_t k_cache_stride;
|
||||
int64_t v_cache_stride;
|
||||
uint32_t num_tokens;
|
||||
uint32_t q_dim;
|
||||
uint32_t kv_dim;
|
||||
@@ -76,12 +79,12 @@ __global__ void fused_fp8_qkv_kv_cache_kernel(const __grid_constant__ FusedQkvPa
|
||||
}
|
||||
quant_row<T, kVecN>(
|
||||
static_cast<const T*>(params.k) + static_cast<size_t>(token) * params.k_stride,
|
||||
static_cast<fp8_e4m3_t*>(params.k_cache) + static_cast<size_t>(slot) * params.kv_dim,
|
||||
static_cast<fp8_e4m3_t*>(params.k_cache) + static_cast<size_t>(slot) * params.k_cache_stride,
|
||||
params.kv_dim,
|
||||
inv_k);
|
||||
quant_row<T, kVecN>(
|
||||
static_cast<const T*>(params.v) + static_cast<size_t>(token) * params.v_stride,
|
||||
static_cast<fp8_e4m3_t*>(params.v_cache) + static_cast<size_t>(slot) * params.kv_dim,
|
||||
static_cast<fp8_e4m3_t*>(params.v_cache) + static_cast<size_t>(slot) * params.v_cache_stride,
|
||||
params.kv_dim,
|
||||
inv_v);
|
||||
|
||||
@@ -127,13 +130,16 @@ struct FusedFp8QkvKvCache {
|
||||
auto S = SymbolicSize{"num_slots"};
|
||||
auto SK = SymbolicSize{"k_stride"};
|
||||
auto SV = SymbolicSize{"v_stride"};
|
||||
auto SKC = SymbolicSize{"k_cache_stride"};
|
||||
auto SVC = SymbolicSize{"v_cache_stride"};
|
||||
auto device = SymbolicDevice{};
|
||||
auto idx_dtype = SymbolicDType{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({N, Dkv}).with_strides({SK, 1}).with_dtype<T>().with_device(device).verify(k);
|
||||
TensorMatcher({N, Dkv}).with_strides({SV, 1}).with_dtype<T>().with_device(device).verify(v);
|
||||
TensorMatcher({S, Dkv}).with_dtype<fp8_e4m3_t>().with_device(device).verify(k_cache).verify(v_cache);
|
||||
TensorMatcher({S, Dkv}).with_strides({SKC, 1}).with_dtype<fp8_e4m3_t>().with_device(device).verify(k_cache);
|
||||
TensorMatcher({S, Dkv}).with_strides({SVC, 1}).with_dtype<fp8_e4m3_t>().with_device(device).verify(v_cache);
|
||||
TensorMatcher({N}).with_dtype<int32_t, int64_t>(idx_dtype).with_device(device).verify(cache_loc);
|
||||
TensorMatcher({1}).with_dtype<fp32_t>().with_device(device).verify(k_scale).verify(v_scale);
|
||||
|
||||
@@ -156,12 +162,17 @@ struct FusedFp8QkvKvCache {
|
||||
const uint32_t kv_dim = static_cast<uint32_t>(Dkv.unwrap());
|
||||
const int64_t k_stride = SK.unwrap();
|
||||
const int64_t v_stride = SV.unwrap();
|
||||
const int64_t k_cache_stride = SKC.unwrap();
|
||||
const int64_t v_cache_stride = SVC.unwrap();
|
||||
RuntimeCheck(num_tokens > 0, "fused_fp8_qkv_kv_cache: num_tokens must be > 0, got ", num_tokens);
|
||||
|
||||
auto fits = [&](int vec) {
|
||||
const int in_bytes = vec * static_cast<int>(sizeof(T));
|
||||
const int out_bytes = vec * static_cast<int>(sizeof(fp8_e4m3_t));
|
||||
bool ok = kv_dim % vec == 0 && k_stride % vec == 0 && v_stride % vec == 0 && aligned(k.data_ptr(), in_bytes) &&
|
||||
aligned(v.data_ptr(), in_bytes);
|
||||
ok = ok && k_cache_stride % vec == 0 && v_cache_stride % vec == 0 && aligned(k_cache.data_ptr(), out_bytes) &&
|
||||
aligned(v_cache.data_ptr(), out_bytes);
|
||||
if (quantize_q) {
|
||||
ok = ok && q_dim % vec == 0 && q_stride % vec == 0 && aligned(q_ptr, in_bytes);
|
||||
}
|
||||
@@ -182,6 +193,8 @@ struct FusedFp8QkvKvCache {
|
||||
.q_stride = q_stride,
|
||||
.k_stride = k_stride,
|
||||
.v_stride = v_stride,
|
||||
.k_cache_stride = k_cache_stride,
|
||||
.v_cache_stride = v_cache_stride,
|
||||
.num_tokens = num_tokens,
|
||||
.q_dim = q_dim,
|
||||
.kv_dim = kv_dim,
|
||||
|
||||
@@ -132,7 +132,7 @@ struct AttnPrologueParams {
|
||||
int64_t inter_stride_b;
|
||||
int64_t inter_stride_t;
|
||||
int64_t inter_stride_w;
|
||||
int64_t kv_buf_stride; // elems per KV slot row (= Hkv * head_dim)
|
||||
int64_t kv_buf_stride; // elems between consecutive KV slots (>= Hkv * head_dim)
|
||||
uint32_t T;
|
||||
uint32_t q; // draft_token_num
|
||||
uint32_t dq;
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace canary {
|
||||
// Device-side handle for one real-KV source.
|
||||
struct RealKvSourceHandle {
|
||||
const uint8_t* tensor; // raw uint8 byte pointer to the source tensor
|
||||
int32_t row_stride_bytes; // tensor.shape[1] in bytes (may exceed page_size * num_bytes_per_token)
|
||||
int32_t row_stride_bytes; // tensor.stride(0) in bytes (may exceed page_size * num_bytes_per_token)
|
||||
int32_t page_size;
|
||||
int32_t num_bytes_per_token;
|
||||
int32_t read_bytes;
|
||||
@@ -37,8 +37,8 @@ SGL_DEVICE uint64_t splitmix64_mix3(uint64_t a, uint64_t b, uint64_t c) {
|
||||
// tensor[slot_idx // page_size,
|
||||
// (slot_idx % page_size) * num_bytes_per_token + byte_offset]
|
||||
//
|
||||
// row_stride_bytes is the dim-1 size of the underlying tensor in bytes (which may exceed
|
||||
// page_size * num_bytes_per_token; trailing bytes are skipped).
|
||||
// row_stride_bytes is the dim-0 stride of the underlying tensor in bytes (which may exceed
|
||||
// page_size * num_bytes_per_token; the remaining bytes are skipped).
|
||||
SGL_DEVICE void real_kv_load_uint4(
|
||||
const RealKvSourceHandle& src, int64_t slot_idx, int64_t byte_offset, uint64_t& word_lo, uint64_t& word_hi) {
|
||||
const int64_t row = slot_idx / src.page_size;
|
||||
|
||||
@@ -205,24 +205,28 @@ struct CanaryVerifyKernel {
|
||||
SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"};
|
||||
SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"};
|
||||
TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_0);
|
||||
SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"};
|
||||
SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"};
|
||||
TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_1);
|
||||
SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"};
|
||||
SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"};
|
||||
TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_2);
|
||||
SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"};
|
||||
SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"};
|
||||
TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_3);
|
||||
@@ -271,7 +275,7 @@ struct CanaryVerifyKernel {
|
||||
tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3};
|
||||
for (int s = 0; s < kMaxRealKvSources; ++s) {
|
||||
p.sources[s].tensor = static_cast<const uint8_t*>(source_bufs[s].data_ptr());
|
||||
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].size(1));
|
||||
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].stride(0));
|
||||
p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize];
|
||||
p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken];
|
||||
p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes];
|
||||
|
||||
@@ -251,24 +251,28 @@ inline void canary_write_step_cuda(
|
||||
SymbolicSize N_real_kv_rows_0 = {"real_kv_rows_0"};
|
||||
SymbolicSize N_real_kv_cols_0 = {"real_kv_cols_0"};
|
||||
TensorMatcher({N_real_kv_rows_0, N_real_kv_cols_0})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_0);
|
||||
SymbolicSize N_real_kv_rows_1 = {"real_kv_rows_1"};
|
||||
SymbolicSize N_real_kv_cols_1 = {"real_kv_cols_1"};
|
||||
TensorMatcher({N_real_kv_rows_1, N_real_kv_cols_1})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_1);
|
||||
SymbolicSize N_real_kv_rows_2 = {"real_kv_rows_2"};
|
||||
SymbolicSize N_real_kv_cols_2 = {"real_kv_cols_2"};
|
||||
TensorMatcher({N_real_kv_rows_2, N_real_kv_cols_2})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_2);
|
||||
SymbolicSize N_real_kv_rows_3 = {"real_kv_rows_3"};
|
||||
SymbolicSize N_real_kv_cols_3 = {"real_kv_cols_3"};
|
||||
TensorMatcher({N_real_kv_rows_3, N_real_kv_cols_3})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<uint8_t>()
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(real_kv_buf_3);
|
||||
@@ -329,7 +333,7 @@ inline void canary_write_step_cuda(
|
||||
tvm::ffi::TensorView source_bufs[kMaxRealKvSources] = {real_kv_buf_0, real_kv_buf_1, real_kv_buf_2, real_kv_buf_3};
|
||||
for (int s = 0; s < kMaxRealKvSources; ++s) {
|
||||
p.sources[s].tensor = static_cast<const uint8_t*>(source_bufs[s].data_ptr());
|
||||
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].size(1));
|
||||
p.sources[s].row_stride_bytes = static_cast<int32_t>(source_bufs[s].stride(0));
|
||||
p.sources[s].page_size = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldPageSize];
|
||||
p.sources[s].num_bytes_per_token = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldNumBytesPerToken];
|
||||
p.sources[s].read_bytes = params[s * kRealKvSourceFieldsPerEntry + kRealKvSourceFieldReadBytes];
|
||||
|
||||
@@ -70,30 +70,21 @@ if _is_cuda:
|
||||
#
|
||||
# See: https://github.com/flashinfer-ai/flashinfer/issues/2232
|
||||
def canonicalize_stride(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Adjust degenerate strides for a tensor, make it canonical.
|
||||
"""Give each size-1 dim whose stride collides with the next dim's a
|
||||
canonical stride; every other stride is kept, so a non-contiguous KV view
|
||||
keeps its addressing.
|
||||
|
||||
Example: shape ``[num_pages, 1, 64, 128]`` with stride
|
||||
``[8192, 128, 128, 1]`` becomes stride ``[8192, 8192, 128, 1]``.
|
||||
"""
|
||||
sizes = tensor.size()
|
||||
strides = tensor.stride()
|
||||
ndim = tensor.dim()
|
||||
|
||||
need_fix = any(
|
||||
sizes[i] == 1 and strides[i] == strides[i + 1] for i in range(ndim - 1)
|
||||
)
|
||||
|
||||
if not need_fix:
|
||||
strides = list(tensor.stride())
|
||||
new_strides = list(strides)
|
||||
for i in range(tensor.dim() - 2, -1, -1):
|
||||
if sizes[i] == 1 and strides[i] == strides[i + 1]:
|
||||
new_strides[i] = new_strides[i + 1] * sizes[i + 1]
|
||||
if new_strides == strides:
|
||||
return tensor
|
||||
|
||||
# canonicalize the stride
|
||||
# Example:
|
||||
# - shape: [num_pages, 1, 64, 128]
|
||||
# - stride: [8192, 128, 128, 1] (wrong!)
|
||||
# Gives new stride: [8192, 8192, 128 ,1] (correct!)
|
||||
new_strides = [0] * ndim
|
||||
new_strides[-1] = 1
|
||||
for i in range(ndim - 2, -1, -1):
|
||||
new_strides[i] = new_strides[i + 1] * sizes[i + 1]
|
||||
|
||||
return tensor.as_strided(sizes, new_strides)
|
||||
|
||||
|
||||
|
||||
@@ -56,22 +56,23 @@ class RealKvSource:
|
||||
num_bytes_per_token``. Trailing bytes of each row are ignored by the canary; this is exactly how the
|
||||
abstraction accommodates pools whose per-row layout interleaves canary-relevant bytes with other metadata
|
||||
(layer-split storage, K/V interleaving, ...). When ``page_size == 1`` the pattern
|
||||
collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case.
|
||||
collapses to the simple ``tensor[slot_idx, :num_bytes_per_token]`` case. Rows are stepped by
|
||||
``tensor.stride(0)``, so dim 0 may be a per-layer view into a larger buffer.
|
||||
|
||||
A pool may expose multiple RealKvSource instances per (canary buffer × K/V half) — the launch wrappers
|
||||
iterate the source list and fold each into the running real_kv_hash via splitmix64 (one int64 fingerprint
|
||||
per slot, regardless of source count).
|
||||
|
||||
Pool patchers construct sources by:
|
||||
- viewing / reshaping the underlying KV layer into the canonical [num_rows, dim1_bytes] form (no stage-copy
|
||||
needed when the underlying storage is already row-major contiguous on dim 0),
|
||||
- viewing the underlying KV layer as [num_rows, dim1_bytes] (a view, never a stage-copy: a copy would
|
||||
fingerprint a stale snapshot instead of the live pool),
|
||||
- choosing ``page_size`` and ``num_bytes_per_token`` so that the access pattern above lands on the bytes
|
||||
the canary should fingerprint,
|
||||
- leaving any per-row padding / non-canary bytes in the trailing portion of each row (they will simply be
|
||||
skipped).
|
||||
|
||||
16-byte alignment precondition: the CUDA fold kernel issues 128-bit aligned loads, so ``read_bytes``,
|
||||
``num_bytes_per_token``, and the row stride (``tensor.shape[1]`` in bytes) must all be positive
|
||||
``num_bytes_per_token``, and the row stride (``tensor.stride(0)`` in bytes) must all be positive
|
||||
multiples of 16. There is no "skip this source" sentinel — callers omit the source from their
|
||||
``real_kv_sources`` tuple entirely (factory helpers return an empty tuple in that case).
|
||||
|
||||
@@ -114,19 +115,21 @@ class RealKvSource:
|
||||
raise ValueError(
|
||||
f"kv-canary: RealKvSource.tensor must be at least 2-D, got shape {tuple(self.tensor.shape)}"
|
||||
)
|
||||
row_stride_bytes = int(self.tensor.shape[1]) * self.tensor.element_size()
|
||||
row_stride_bytes = int(self.tensor.stride(0)) * self.tensor.element_size()
|
||||
if row_stride_bytes % 16 != 0:
|
||||
raise ValueError(
|
||||
f"kv-canary: RealKvSource.tensor dim-1 byte width must be a multiple of 16, "
|
||||
f"kv-canary: RealKvSource.tensor row stride must be a multiple of 16 bytes, "
|
||||
f"got {row_stride_bytes} bytes (shape={tuple(self.tensor.shape)}, "
|
||||
f"dtype={self.tensor.dtype})"
|
||||
f"strides={tuple(self.tensor.stride())}, dtype={self.tensor.dtype})"
|
||||
)
|
||||
# A row is addressed as page_size slots of num_bytes_per_token, unchecked at fold time;
|
||||
# a narrower row hashes fewer bytes than asked and still reports the chain clean.
|
||||
# a narrower row hashes fewer bytes than asked and still reports the chain clean. Check the
|
||||
# row's width, not its stride.
|
||||
row_bytes = int(self.tensor.shape[1]) * self.tensor.element_size()
|
||||
min_row_bytes = self.page_size * self.num_bytes_per_token
|
||||
if row_stride_bytes < min_row_bytes:
|
||||
if row_bytes < min_row_bytes:
|
||||
raise ValueError(
|
||||
f"kv-canary: RealKvSource.tensor dim-1 is {row_stride_bytes} bytes but "
|
||||
f"kv-canary: RealKvSource.tensor row is {row_bytes} bytes wide but "
|
||||
f"page_size={self.page_size} x num_bytes_per_token={self.num_bytes_per_token} "
|
||||
f"needs {min_row_bytes} (shape={tuple(self.tensor.shape)}, "
|
||||
f"dtype={self.tensor.dtype})"
|
||||
@@ -412,7 +415,12 @@ def _build_real_kv_source_abi(
|
||||
)
|
||||
|
||||
for i, source in enumerate(real_kv_sources):
|
||||
_assert_contiguous(source.tensor, f"real_kv_sources[{i}].tensor")
|
||||
# The kernels step rows by stride(0), so only the bytes within a row must be packed.
|
||||
if source.tensor.stride(-1) != 1:
|
||||
raise ValueError(
|
||||
f"kv-canary: real_kv_sources[{i}].tensor rows must be packed "
|
||||
f"(stride(-1) == 1), got strides {tuple(source.tensor.stride())}"
|
||||
)
|
||||
source_u8 = source.tensor.view(torch.uint8)
|
||||
if source_u8.dim() != 2:
|
||||
raise ValueError(
|
||||
|
||||
@@ -67,11 +67,10 @@ def make_row_source(
|
||||
layer_buffer: torch.Tensor,
|
||||
read_bytes: int,
|
||||
) -> Tuple[RealKvSource, ...]:
|
||||
contiguous = layer_buffer.contiguous()
|
||||
num_slots = int(contiguous.shape[0])
|
||||
num_slots = int(layer_buffer.shape[0])
|
||||
if num_slots == 0 or read_bytes == 0:
|
||||
return ()
|
||||
flat = contiguous.view(torch.uint8).reshape(num_slots, -1)
|
||||
flat = layer_buffer.view(torch.uint8).view(num_slots, -1)
|
||||
num_bytes_per_token = int(flat.shape[1])
|
||||
clipped = _clip_read_bytes_aligned(
|
||||
requested=read_bytes, num_bytes_per_token=num_bytes_per_token
|
||||
|
||||
@@ -2117,6 +2117,13 @@ class MHATokenToKVPool(KVCache):
|
||||
self._kv_copy_config = None
|
||||
return
|
||||
|
||||
# The tiled copy uses `data_strides[i]` as both slot stride and copy width.
|
||||
for buf in self._slot_move_pointer_buffers():
|
||||
assert buf.is_contiguous(), (
|
||||
"the tiled KV copy needs contiguous KV buffers; got shape "
|
||||
f"{tuple(buf.shape)}, strides {buf.stride()}"
|
||||
)
|
||||
|
||||
# Heuristics for KV copy tiling
|
||||
_KV_COPY_STRIDE_THRESHOLD_LARGE = 8192
|
||||
_KV_COPY_STRIDE_THRESHOLD_MEDIUM = 4096
|
||||
@@ -2292,10 +2299,7 @@ class MHATokenToKVPool(KVCache):
|
||||
device=self.device,
|
||||
)
|
||||
self.data_strides = torch.tensor(
|
||||
[
|
||||
np.prod(x.shape[1:]) * x.dtype.itemsize
|
||||
for x in slot_move_pointer_buffers
|
||||
],
|
||||
[x.stride(0) * x.dtype.itemsize for x in slot_move_pointer_buffers],
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
@@ -2647,11 +2651,13 @@ class MHATokenToKVPool(KVCache):
|
||||
|
||||
if dcp_kv_mask is not None:
|
||||
N, H, D = cache_k.shape
|
||||
k_buf = self.k_buffer[layer_id - self.start_layer]
|
||||
v_buf = self.v_buffer[layer_id - self.start_layer]
|
||||
masked_set_kv_buffer_kernel[(N,)](
|
||||
cache_k,
|
||||
cache_v,
|
||||
self.k_buffer[layer_id - self.start_layer],
|
||||
self.v_buffer[layer_id - self.start_layer],
|
||||
k_buf,
|
||||
v_buf,
|
||||
loc,
|
||||
dcp_kv_mask,
|
||||
N,
|
||||
@@ -2662,6 +2668,8 @@ class MHATokenToKVPool(KVCache):
|
||||
cache_k.stride(1),
|
||||
cache_v.stride(0),
|
||||
cache_v.stride(1),
|
||||
k_buf.stride(0),
|
||||
v_buf.stride(0),
|
||||
)
|
||||
return
|
||||
|
||||
@@ -3224,10 +3232,7 @@ class NoOpMHATokenToKVPool(MHATokenToKVPool):
|
||||
)
|
||||
self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0)
|
||||
self.data_strides = torch.tensor(
|
||||
[
|
||||
np.prod(x.shape[1:]) * x.dtype.itemsize
|
||||
for x in self.k_buffer + self.v_buffer
|
||||
],
|
||||
[x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer],
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
@@ -3610,10 +3615,7 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
|
||||
)
|
||||
self.data_ptrs = torch.cat([self.k_data_ptrs, self.v_data_ptrs], dim=0)
|
||||
self.data_strides = torch.tensor(
|
||||
[
|
||||
np.prod(x.shape[1:]) * x.dtype.itemsize
|
||||
for x in self.k_buffer + self.v_buffer
|
||||
],
|
||||
[x.stride(0) * x.dtype.itemsize for x in self.k_buffer + self.v_buffer],
|
||||
device=self.device,
|
||||
)
|
||||
# This override replaces the base allocation, so the PD-transfer
|
||||
@@ -5295,6 +5297,8 @@ def masked_set_kv_buffer_kernel(
|
||||
k_stride_H: tl.constexpr,
|
||||
v_stride_B: tl.constexpr,
|
||||
v_stride_H: tl.constexpr,
|
||||
k_buffer_stride: tl.constexpr,
|
||||
v_buffer_stride: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
if pid >= N:
|
||||
@@ -5316,10 +5320,10 @@ def masked_set_kv_buffer_kernel(
|
||||
col = idx % D
|
||||
|
||||
key = tl.load(k_ptr + pid * k_stride_B + row * k_stride_H + col, mask=mask)
|
||||
tl.store(k_buffer_ptr + loc * H * D + idx, key, mask=mask)
|
||||
tl.store(k_buffer_ptr + loc * k_buffer_stride + idx, key, mask=mask)
|
||||
|
||||
value = tl.load(v_ptr + pid * v_stride_B + row * v_stride_H + col, mask=mask)
|
||||
tl.store(v_buffer_ptr + loc * H * D + idx, value, mask=mask)
|
||||
tl.store(v_buffer_ptr + loc * v_buffer_stride + idx, value, mask=mask)
|
||||
|
||||
|
||||
class MHATokenToKOnlyPool(KVCache):
|
||||
|
||||
@@ -278,6 +278,54 @@ class HostKVCache(abc.ABC):
|
||||
_cuda_host_unregister(buf)
|
||||
self.kv_buffer = None
|
||||
|
||||
# Bytes between strided device KV rows; None (the default, and what a pool
|
||||
# built without `_init_device_row_stride` keeps) means packed rows.
|
||||
_strided_device_row_bytes: Optional[int] = None
|
||||
|
||||
def _init_device_row_stride(self, device_buffers: Iterable[torch.Tensor]) -> None:
|
||||
"""Record the row stride of strided device KV buffers.
|
||||
|
||||
A strided per-layer view -- the unified pool's token-major entries --
|
||||
steps its rows by `stride(0)` rather than by the row width, and the
|
||||
device-side addresses in the transfer kernels must follow it.
|
||||
"""
|
||||
buffers = list(device_buffers)
|
||||
strides = {
|
||||
buf.stride(0) * buf.element_size()
|
||||
for buf in buffers
|
||||
if not buf.is_contiguous()
|
||||
}
|
||||
if not strides:
|
||||
return
|
||||
if len(strides) > 1 or any(buf.is_contiguous() for buf in buffers):
|
||||
raise NotImplementedError(
|
||||
"HiCache: the device KV buffers do not share one row stride "
|
||||
f"(strided rows {sorted(strides)} B, mixed with packed ones)"
|
||||
)
|
||||
for buf in buffers:
|
||||
assert buf.stride(-1) == 1, (
|
||||
f"HiCache needs packed device rows; got strides {buf.stride()}"
|
||||
)
|
||||
self._strided_device_row_bytes = strides.pop()
|
||||
|
||||
@property
|
||||
def device_rows_packed(self) -> bool:
|
||||
return self._strided_device_row_bytes is None
|
||||
|
||||
@property
|
||||
def device_row_stride_bytes(self) -> int:
|
||||
"""Bytes between consecutive device KV rows."""
|
||||
return self._strided_device_row_bytes or self.token_stride_size
|
||||
|
||||
def _require_packed_device_rows(self, kernel: str) -> None:
|
||||
if not self.device_rows_packed:
|
||||
raise NotImplementedError(
|
||||
f"HiCache: {kernel} steps device rows by the row width "
|
||||
f"({self.token_stride_size} B), but this device pool's rows are "
|
||||
f"{self.device_row_stride_bytes} B apart. Use the JIT HiCache "
|
||||
"kernels with a layer_first or page_first host layout."
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_size_per_token(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -152,6 +152,13 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
pool_label=pool_label,
|
||||
)
|
||||
self.element_dim = self.head_num * self.head_dim
|
||||
self._init_device_row_stride(
|
||||
buf
|
||||
for pool in (self.device_pool, *self.mtp_draft_device_pools)
|
||||
if pool is not None
|
||||
for side in ("k_buffer", "v_buffer")
|
||||
for buf in getattr(pool, side, None) or ()
|
||||
)
|
||||
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
|
||||
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
|
||||
# write-back kernel has a ROCm path, so enable them on HIP too. This
|
||||
@@ -268,7 +275,8 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
self.staging_k_buffer = None
|
||||
self.staging_v_buffer = None
|
||||
self.can_use_write_back_jit = False
|
||||
if self.layout != "page_first":
|
||||
# The staged kernel reads whole device pages, so it needs packed rows.
|
||||
if self.layout != "page_first" or not self.device_rows_packed:
|
||||
return
|
||||
page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK)
|
||||
staging = prepare_mha_write_back_staging(
|
||||
@@ -328,6 +336,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
element_dim=self.element_dim,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_per_layer")
|
||||
transfer_kv_per_layer(
|
||||
src_k=self.k_buffer[host_layer_id],
|
||||
dst_k=device_pool.k_buffer[device_layer_id],
|
||||
@@ -353,6 +362,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
element_dim=self.element_dim,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_pf_lf")
|
||||
transfer_kv_per_layer_pf_lf(
|
||||
src_k=self.k_buffer,
|
||||
dst_k=device_pool.k_buffer[device_layer_id],
|
||||
@@ -365,6 +375,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
src_layout_dim=self.layout_dim,
|
||||
)
|
||||
elif self.layout == "page_head":
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_ph_lf")
|
||||
transfer_kv_per_layer_ph_lf(
|
||||
src_k=self.k_buffer,
|
||||
dst_k=device_pool.k_buffer[device_layer_id],
|
||||
@@ -489,10 +500,11 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
v_ptr_src=device_v_data_ptrs,
|
||||
indices_src=device_indices,
|
||||
kv_cache_dst_stride_bytes=self.token_stride_size,
|
||||
kv_cache_src_stride_bytes=self.token_stride_size,
|
||||
kv_cache_src_stride_bytes=self.device_row_stride_bytes,
|
||||
element_size=self.element_dim * self.dtype.itemsize,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_all_layer")
|
||||
transfer_kv_all_layer(
|
||||
src_k_layers=device_k_data_ptrs,
|
||||
dst_k_layers=self.k_data_ptrs,
|
||||
@@ -516,7 +528,23 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
dst_v=self.v_buffer,
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.can_use_jit and not self.device_rows_packed:
|
||||
# The per-layer host views of page_first rows sit
|
||||
# `layout_dim` apart, which the all-layer kernel can step.
|
||||
jit_transfer_hicache_all_layer(
|
||||
page_size=self.page_size,
|
||||
k_ptr_dst=self.k_data_ptrs,
|
||||
v_ptr_dst=self.v_data_ptrs,
|
||||
indices_dst=host_indices,
|
||||
k_ptr_src=device_k_data_ptrs,
|
||||
v_ptr_src=device_v_data_ptrs,
|
||||
indices_src=device_indices,
|
||||
kv_cache_dst_stride_bytes=self.layout_dim,
|
||||
kv_cache_src_stride_bytes=self.device_row_stride_bytes,
|
||||
element_size=self.element_dim * self.dtype.itemsize,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_lf_pf")
|
||||
transfer_kv_all_layer_lf_pf(
|
||||
src_k_layers=device_k_data_ptrs,
|
||||
dst_k=self.k_buffer,
|
||||
@@ -529,6 +557,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
num_layers=self.layer_num,
|
||||
)
|
||||
elif self.layout == "page_head":
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_lf_ph")
|
||||
transfer_kv_all_layer_lf_ph(
|
||||
src_k_layers=device_k_data_ptrs,
|
||||
dst_k=self.k_buffer,
|
||||
|
||||
@@ -148,6 +148,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
self.packed_device_kv_buffers = [
|
||||
buffer for pool in device_pools for buffer in pool.kv_buffer
|
||||
]
|
||||
self._init_device_row_stride(
|
||||
buf
|
||||
for pool in (self.device_pool, *self.mtp_draft_device_pools)
|
||||
for buf in getattr(pool, "kv_buffer", None) or ()
|
||||
)
|
||||
self._init_write_back_staging_buffers()
|
||||
|
||||
def _init_dummy(
|
||||
@@ -402,7 +407,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
self.staging_token_capacity = 0
|
||||
self.staging_buffer = None
|
||||
self.can_use_write_back_jit = False
|
||||
if self.layout != "page_first" or (_is_npu or _is_xpu or _is_mps):
|
||||
# The staged kernel reads whole device pages, so it needs packed rows.
|
||||
if (
|
||||
self.layout != "page_first"
|
||||
or not self.device_rows_packed
|
||||
or (_is_npu or _is_xpu or _is_mps)
|
||||
):
|
||||
return
|
||||
|
||||
# The staged write-back JIT kernel builds with hipcc and has a ROCm
|
||||
@@ -680,6 +690,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
element_dim=self.kv_cache_dim,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_mla")
|
||||
transfer_kv_per_layer_mla(
|
||||
src=self.kv_buffer[host_layer_id],
|
||||
dst=device_pool.kv_buffer[device_layer_id],
|
||||
@@ -698,6 +709,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
element_dim=self.kv_cache_dim,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_mla_pf_lf")
|
||||
transfer_kv_per_layer_mla_pf_lf(
|
||||
src=self.kv_buffer,
|
||||
dst=device_pool.kv_buffer[device_layer_id],
|
||||
@@ -808,6 +820,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
element_dim=self.kv_cache_dim,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_mla")
|
||||
transfer_kv_per_layer_mla(
|
||||
src=device_pool.kv_buffer[device_layer_id],
|
||||
dst=self.kv_buffer[host_layer_id],
|
||||
@@ -907,10 +920,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
ptr_src=device_data_ptrs,
|
||||
indices_src=device_indices,
|
||||
cache_dst_stride_bytes=self.token_stride_size,
|
||||
cache_src_stride_bytes=self.token_stride_size,
|
||||
cache_src_stride_bytes=self.device_row_stride_bytes,
|
||||
element_size=self.kv_cache_dim * self.dtype.itemsize,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_mla")
|
||||
transfer_kv_all_layer_mla(
|
||||
src_layers=device_data_ptrs,
|
||||
dst_layers=self.data_ptrs,
|
||||
@@ -929,7 +943,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
dst=self.kv_buffer,
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.can_use_jit and not self.device_rows_packed:
|
||||
# The per-layer host views of page_first rows sit
|
||||
# `layout_dim` apart, which the all-layer kernel can step.
|
||||
jit_transfer_hicache_all_layer_mla(
|
||||
page_size=self.page_size,
|
||||
ptr_dst=self.data_ptrs,
|
||||
indices_dst=host_indices,
|
||||
ptr_src=device_data_ptrs,
|
||||
indices_src=device_indices,
|
||||
cache_dst_stride_bytes=self.layout_dim,
|
||||
cache_src_stride_bytes=self.device_row_stride_bytes,
|
||||
element_size=self.kv_cache_dim * self.dtype.itemsize,
|
||||
)
|
||||
else:
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_mla_lf_pf")
|
||||
transfer_kv_all_layer_mla_lf_pf(
|
||||
src_layers=device_data_ptrs,
|
||||
dst=self.kv_buffer,
|
||||
|
||||
@@ -149,6 +149,34 @@ def _rel_proj_kernel_eligible(r: torch.Tensor) -> bool:
|
||||
)
|
||||
|
||||
|
||||
# Slot-stride alignment (elements) of the fused prologue store: kVecElems for
|
||||
# bf16, kMXFP8Block for MXFP8.
|
||||
_FUSED_KV_STORE_SLOT_ALIGN = {torch.bfloat16: 8, torch.float8_e4m3fn: 32}
|
||||
|
||||
|
||||
def _fused_kv_store_eligible(
|
||||
k_buf: torch.Tensor, v_buf: torch.Tensor, head_dim: int, dtype: torch.dtype
|
||||
) -> bool:
|
||||
"""Whether the fused prologue can store K/V of `dtype` straight into these
|
||||
pool buffers.
|
||||
|
||||
It writes one NHD ``[slot, head, head_dim]`` row per loc, stepping slots by
|
||||
the buffers' shared ``stride(0)``, so a strided per-layer view qualifies as
|
||||
long as each row is contiguous. Other layouts (HND / vectorized_5d) keep the
|
||||
backend store, which owns their quant and layout; conv, windows and qk-norm
|
||||
stay fused either way.
|
||||
"""
|
||||
slot_stride = k_buf.stride(0)
|
||||
return slot_stride % _FUSED_KV_STORE_SLOT_ALIGN[dtype] == 0 and all(
|
||||
buf.dtype == dtype
|
||||
and buf.dim() == 3
|
||||
and buf.shape[-1] == head_dim
|
||||
and buf.stride()[1:] == (head_dim, 1)
|
||||
and buf.stride(0) == slot_stride
|
||||
for buf in (k_buf, v_buf)
|
||||
)
|
||||
|
||||
|
||||
class RelLogitsProj(nn.Module):
|
||||
def __init__(self, d_rel: int, rel_extent: int, *, deterministic: bool = False):
|
||||
super().__init__()
|
||||
@@ -438,16 +466,8 @@ class InklingAttention(nn.Module):
|
||||
pool = get_token_to_kv_pool()
|
||||
k_buf = pool.get_key_buffer(self.layer_id)
|
||||
v_buf = pool.get_value_buffer(self.layer_id)
|
||||
# The fused store writes raw bf16 into an NHD [slot, head, head_dim]
|
||||
# buffer indexed by loc. Take it only for that exact layout: FP8/MXFP8
|
||||
# (non-bf16) and HND/vectorized_5d (4D/5D, paged (page, head) index)
|
||||
# pools keep the backend store, which owns their quant + layout. conv +
|
||||
# windows + qk-norm stay fused regardless.
|
||||
do_bf16_store = (
|
||||
k_buf.dtype == torch.bfloat16
|
||||
and k_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
do_bf16_store = _fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.bfloat16
|
||||
)
|
||||
sfk = sfv = None
|
||||
do_mxfp8_store = False
|
||||
@@ -456,14 +476,9 @@ class InklingAttention(nn.Module):
|
||||
):
|
||||
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
|
||||
do_mxfp8_store = (
|
||||
k_buf.dtype == torch.float8_e4m3fn
|
||||
and v_buf.dtype == torch.float8_e4m3fn
|
||||
and k_buf.dim() == 3
|
||||
and v_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and v_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
and v_buf.is_contiguous()
|
||||
_fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
|
||||
)
|
||||
and sfk.dim() == 5
|
||||
and sfv.dim() == 5
|
||||
and getattr(pool, "page_size", 0) == 128
|
||||
@@ -553,15 +568,8 @@ class InklingAttention(nn.Module):
|
||||
pool = get_token_to_kv_pool()
|
||||
k_buf = pool.get_key_buffer(self.layer_id)
|
||||
v_buf = pool.get_value_buffer(self.layer_id)
|
||||
do_bf16_store = (
|
||||
k_buf.dtype == torch.bfloat16
|
||||
and v_buf.dtype == torch.bfloat16
|
||||
and k_buf.dim() == 3
|
||||
and v_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and v_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
and v_buf.is_contiguous()
|
||||
do_bf16_store = _fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.bfloat16
|
||||
)
|
||||
sfk = sfv = None
|
||||
do_mxfp8_store = False
|
||||
@@ -570,14 +578,9 @@ class InklingAttention(nn.Module):
|
||||
):
|
||||
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
|
||||
do_mxfp8_store = (
|
||||
k_buf.dtype == torch.float8_e4m3fn
|
||||
and v_buf.dtype == torch.float8_e4m3fn
|
||||
and k_buf.dim() == 3
|
||||
and v_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and v_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
and v_buf.is_contiguous()
|
||||
_fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
|
||||
)
|
||||
and sfk.dim() == 5
|
||||
and sfv.dim() == 5
|
||||
and getattr(pool, "page_size", 0) == 128
|
||||
@@ -672,15 +675,8 @@ class InklingAttention(nn.Module):
|
||||
pool = get_token_to_kv_pool()
|
||||
k_buf = pool.get_key_buffer(self.layer_id)
|
||||
v_buf = pool.get_value_buffer(self.layer_id)
|
||||
do_bf16_store = (
|
||||
k_buf.dtype == torch.bfloat16
|
||||
and v_buf.dtype == torch.bfloat16
|
||||
and k_buf.dim() == 3
|
||||
and v_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and v_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
and v_buf.is_contiguous()
|
||||
do_bf16_store = _fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.bfloat16
|
||||
)
|
||||
sfk = sfv = None
|
||||
do_mxfp8_store = False
|
||||
@@ -689,14 +685,9 @@ class InklingAttention(nn.Module):
|
||||
):
|
||||
sfk, sfv = pool.get_kv_scale_buffer(self.layer_id)
|
||||
do_mxfp8_store = (
|
||||
k_buf.dtype == torch.float8_e4m3fn
|
||||
and v_buf.dtype == torch.float8_e4m3fn
|
||||
and k_buf.dim() == 3
|
||||
and v_buf.dim() == 3
|
||||
and k_buf.shape[-1] == self.head_dim
|
||||
and v_buf.shape[-1] == self.head_dim
|
||||
and k_buf.is_contiguous()
|
||||
and v_buf.is_contiguous()
|
||||
_fused_kv_store_eligible(
|
||||
k_buf, v_buf, self.head_dim, torch.float8_e4m3fn
|
||||
)
|
||||
and sfk.dim() == 5
|
||||
and sfv.dim() == 5
|
||||
and getattr(pool, "page_size", 0) == 128
|
||||
|
||||
@@ -83,6 +83,7 @@ def make_real_kv_source(
|
||||
page_size: int = 1,
|
||||
read_bytes: Optional[int] = None,
|
||||
pad_dim1: int = 0,
|
||||
row_gap: int = 0,
|
||||
device: torch.device,
|
||||
fill: int = 0,
|
||||
) -> RealKvSource:
|
||||
@@ -90,12 +91,18 @@ def make_real_kv_source(
|
||||
|
||||
``pad_dim1`` adds trailing per-row bytes the canary should skip — used by the "holey dim 1" case to
|
||||
confirm the kernel never reads past ``page_size * num_bytes_per_token``.
|
||||
|
||||
``row_gap`` makes the source a strided view: each row is followed by ``row_gap`` bytes that belong
|
||||
to the backing buffer but not to the view (filled with the complement of ``fill``), the way a
|
||||
per-layer view into a larger KV entry is laid out. The kernel must step rows by ``stride(0)``.
|
||||
"""
|
||||
num_rows = (num_slots + page_size - 1) // page_size
|
||||
cols = page_size * num_bytes_per_token + pad_dim1
|
||||
tensor = torch.full(
|
||||
(num_rows, cols), fill_value=fill, dtype=torch.uint8, device=device
|
||||
backing = torch.full(
|
||||
(num_rows, cols + row_gap), fill_value=fill, dtype=torch.uint8, device=device
|
||||
)
|
||||
backing[:, cols:] = ~fill & 0xFF
|
||||
tensor = backing[:, :cols]
|
||||
effective_read = read_bytes if read_bytes is not None else num_bytes_per_token
|
||||
return RealKvSource(
|
||||
tensor=tensor,
|
||||
@@ -117,6 +124,7 @@ def make_real_kv_sources(
|
||||
device: torch.device,
|
||||
rng: Optional[random.Random] = None,
|
||||
fill_strategy: FillStrategy = "constant_per_source",
|
||||
row_gap: int = 0,
|
||||
) -> tuple[RealKvSource, ...]:
|
||||
sources: list[RealKvSource] = []
|
||||
for i in range(count):
|
||||
@@ -126,6 +134,7 @@ def make_real_kv_sources(
|
||||
num_bytes_per_token=num_bytes_per_token,
|
||||
page_size=page_size,
|
||||
read_bytes=read_bytes_eff,
|
||||
row_gap=row_gap,
|
||||
device=device,
|
||||
fill=(i + 1) * 17,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user