[AMD] [GLM-5.3-Flash Day 0] Enable the k-pool DSA indexer on gfx950 (#39341)
Co-authored-by: Yikai Zhang <ykzhang12@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com> Co-authored-by: Thomas Wang <thomawan@amd.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
This commit is contained in:
co-authored by
Yikai Zhang
Claude Opus 5
Raiden-Makoto
Thomas Wang
Kevin Mi
Kevin Mi
parent
077c319914
commit
3afdde5f05
@@ -14,7 +14,18 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import (
|
||||
rotate_activation,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod
|
||||
from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp
|
||||
from sglang.srt.layers.attention.dsa.paged_mqa_logits_backend import (
|
||||
DSAPagedMQALogitsBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
dsa_use_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.attention.mqa_logits_utils import (
|
||||
MQA_LOGITS_BYTES_PER_ELEM,
|
||||
MQA_LOGITS_MAX_BYTES_ROCM,
|
||||
mqa_logits_rows_per_chunk,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import LayerNorm
|
||||
from sglang.srt.layers.utils import MultiPlatformOp
|
||||
from sglang.srt.utils import add_prefix, ceil_align, is_cuda, is_hip, is_npu
|
||||
@@ -46,7 +57,7 @@ from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_device
|
||||
from sglang.srt.runtime_context import get_device, get_exec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
@@ -110,6 +121,11 @@ class IndexerKPool(MultiPlatformOp):
|
||||
f"index_kpool ({self.index_kpool}) must divide page_size (64)"
|
||||
)
|
||||
|
||||
# resolve() pins ROCm to AITER and lets --dsa-paged-mqa-logits-backend reach this path
|
||||
self.paged_mqa_logits_backend = DSAPagedMQALogitsBackend.resolve(
|
||||
get_exec().kernel.dsa_paged_mqa_logits_backend
|
||||
)
|
||||
|
||||
self.index_kpool_compress_ape = nn.Parameter(
|
||||
torch.zeros(self.index_kpool, self.head_dim, dtype=torch.float32)
|
||||
)
|
||||
@@ -808,12 +824,66 @@ class IndexerKPool(MultiPlatformOp):
|
||||
return None, attn_metadata.topk_indices_offset, None
|
||||
return None, None, None
|
||||
|
||||
@staticmethod
|
||||
def _fp8_mqa_logits(
|
||||
q_fp8: torch.Tensor,
|
||||
k_fp8: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
starts: torch.Tensor,
|
||||
ends: torch.Tensor,
|
||||
*,
|
||||
clean_logits: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Ragged MQA logits: DeepGEMM on CUDA, AITER's Triton kernel on ROCm."""
|
||||
if is_hip():
|
||||
from aiter.ops.triton.fp8_mqa_logits import fp8_mqa_logits
|
||||
|
||||
# above 2 GiB of fp32 logits the kernel miscompiles and abort()s the
|
||||
# process (ROCm/aiter#5114); rows are independent, so block the queries
|
||||
num_q, num_k = q_fp8.shape[0], k_fp8.shape[0]
|
||||
rows_per_call = mqa_logits_rows_per_chunk(
|
||||
num_rows=num_q,
|
||||
row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM,
|
||||
budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM,
|
||||
)
|
||||
if rows_per_call is None:
|
||||
return fp8_mqa_logits(
|
||||
q_fp8,
|
||||
k_fp8,
|
||||
k_scale,
|
||||
weights,
|
||||
starts,
|
||||
ends,
|
||||
clean_logits=clean_logits,
|
||||
)
|
||||
logits = torch.empty(
|
||||
(num_q, num_k), dtype=torch.float32, device=q_fp8.device
|
||||
)
|
||||
for i in range(0, num_q, rows_per_call):
|
||||
rows = slice(i, i + rows_per_call)
|
||||
logits[rows] = fp8_mqa_logits(
|
||||
q_fp8[rows],
|
||||
k_fp8,
|
||||
k_scale,
|
||||
weights[rows],
|
||||
starts[rows],
|
||||
ends[rows],
|
||||
clean_logits=clean_logits,
|
||||
)
|
||||
return logits
|
||||
|
||||
return deep_gemm.fp8_mqa_logits(
|
||||
q_fp8, (k_fp8, k_scale), weights, starts, ends, clean_logits=clean_logits
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _should_use_tilelang_paged_mqa_logits(q_fp8: torch.Tensor) -> bool:
|
||||
# q_fp8 is [tokens, heads, dim], without the next_n dim
|
||||
if not is_cuda():
|
||||
return False
|
||||
arch_major, _ = torch.cuda.get_device_capability(q_fp8.device)
|
||||
num_heads = q_fp8.shape[2]
|
||||
num_heads = q_fp8.shape[1]
|
||||
return arch_major == 9 and num_heads not in (32, 64)
|
||||
|
||||
def _get_topk_paged(
|
||||
@@ -850,7 +920,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if n_real < num_q_padded:
|
||||
q_fp8 = q_fp8[:n_real]
|
||||
weights = weights[:n_real]
|
||||
q_fp8 = q_fp8.unsqueeze(1) # the next_n dim is 1 now
|
||||
# aiter_paged_mqa_logits adds the next_n dim itself; the other two want it already there
|
||||
assert len(kv_cache_fp8.shape) == 2
|
||||
block_kv = 64
|
||||
num_heads_kv = 1
|
||||
@@ -860,7 +930,11 @@ class IndexerKPool(MultiPlatformOp):
|
||||
)
|
||||
assert len(weights.shape) == 3
|
||||
weights = weights.squeeze(2)
|
||||
use_tilelang_paged_mqa = self._should_use_tilelang_paged_mqa_logits(q_fp8)
|
||||
use_aiter_paged_mqa = self.paged_mqa_logits_backend.is_aiter()
|
||||
use_tilelang_paged_mqa = (
|
||||
not use_aiter_paged_mqa
|
||||
and self._should_use_tilelang_paged_mqa_logits(q_fp8)
|
||||
)
|
||||
|
||||
pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = (
|
||||
self._get_kpool_decode_metadata(
|
||||
@@ -868,17 +942,40 @@ class IndexerKPool(MultiPlatformOp):
|
||||
block_tables,
|
||||
seqlens_32,
|
||||
blocksize,
|
||||
build_schedule_metadata=not use_tilelang_paged_mqa,
|
||||
build_schedule_metadata=not (
|
||||
use_aiter_paged_mqa or use_tilelang_paged_mqa
|
||||
),
|
||||
)
|
||||
)
|
||||
pool_max_seq_len = pool_block_tables.shape[1] * blocksize
|
||||
if use_tilelang_paged_mqa:
|
||||
if use_aiter_paged_mqa:
|
||||
# kpool_fp8_index writes in AITER's tile order, so a row-major reader would misread the cache
|
||||
if not aiter_can_use_preshuffle_paged_mqa():
|
||||
raise RuntimeError(
|
||||
"The ROCm k-pool indexer needs AITER's preshuffle paged-MQA "
|
||||
"kernel. It requires Triton >= 3.5.0, or "
|
||||
"AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1 with the AOT gluon "
|
||||
"artifacts present in the image."
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsa import aiter_paged_mqa_logits
|
||||
|
||||
logits = aiter_paged_mqa_logits(
|
||||
q_fp8,
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
pool_seqlens,
|
||||
pool_block_tables,
|
||||
pool_max_seq_len,
|
||||
preshuffle=True,
|
||||
kv_block_size=block_kv,
|
||||
)
|
||||
elif use_tilelang_paged_mqa:
|
||||
from sglang.kernels.ops.attention.dsa.tilelang_kernel import (
|
||||
tilelang_fp8_paged_mqa_logits,
|
||||
)
|
||||
|
||||
logits = tilelang_fp8_paged_mqa_logits(
|
||||
q_fp8,
|
||||
q_fp8.unsqueeze(1),
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
pool_seqlens,
|
||||
@@ -889,7 +986,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
)
|
||||
else:
|
||||
logits = deep_gemm.fp8_paged_mqa_logits(
|
||||
q_fp8,
|
||||
q_fp8.unsqueeze(1),
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
pool_context_lens,
|
||||
@@ -967,9 +1064,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
scale_out=k_scale,
|
||||
)
|
||||
k_fp8 = k_u8.view(torch.float8_e4m3fn)
|
||||
logits = deep_gemm.fp8_mqa_logits(
|
||||
logits = self._fp8_mqa_logits(
|
||||
q_fp8[:n_real].contiguous(),
|
||||
(k_fp8.contiguous(), k_scale.contiguous()),
|
||||
k_fp8.contiguous(),
|
||||
k_scale.contiguous(),
|
||||
weights[:n_real].contiguous(),
|
||||
ks_per_q,
|
||||
ke_per_q,
|
||||
@@ -1199,9 +1297,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
and zero_starts_by_batch[i] is not None
|
||||
else torch.zeros((q_len,), dtype=torch.int32, device=q_fp8.device)
|
||||
)
|
||||
local_logits = deep_gemm.fp8_mqa_logits(
|
||||
local_logits = self._fp8_mqa_logits(
|
||||
q_fp8[q_slice].contiguous(),
|
||||
(k_fp8.contiguous(), k_scale.contiguous()),
|
||||
k_fp8.contiguous(),
|
||||
k_scale.contiguous(),
|
||||
weights[q_slice].contiguous(),
|
||||
row_starts,
|
||||
local_pool_lens,
|
||||
@@ -1575,7 +1674,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if not return_indices:
|
||||
return None
|
||||
|
||||
if is_cuda():
|
||||
if is_cuda() or is_hip():
|
||||
if (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
@@ -1603,5 +1702,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
kpool_extend_cache=kpool_extend_cache,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("kpool indexer is only supported on CUDA")
|
||||
raise NotImplementedError(
|
||||
"kpool indexer is only supported on CUDA and ROCm"
|
||||
)
|
||||
return topk_result
|
||||
|
||||
@@ -4,11 +4,50 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
INDEXER_K_CACHE_PRESHUFFLE_TILE,
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
)
|
||||
|
||||
BLOCK_SIZE_K = 64
|
||||
INDEX_HEAD_DIM = 128
|
||||
KPOOL_SCORE_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
|
||||
|
||||
|
||||
def _preshuffle_tile() -> int:
|
||||
"""Tile width for the index-K cache, or 0 to keep the row-major layout."""
|
||||
return (
|
||||
INDEXER_K_CACHE_PRESHUFFLE_TILE if aiter_can_use_preshuffle_paged_mqa() else 0
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kpool_cache_k_offsets(
|
||||
page,
|
||||
slot,
|
||||
cols,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
):
|
||||
"""Offsets of one pooled token's index-K row; must match `_set_k_and_s_triton_kernel`,
|
||||
which lays out the non-pooled cache for the same aiter kernel."""
|
||||
if PRESHUFFLE_TILE:
|
||||
tile = PRESHUFFLE_TILE
|
||||
token_tile_id = slot // tile
|
||||
token_in_tile = slot % tile
|
||||
col_tile_id = cols // tile
|
||||
col_in_tile = cols % tile
|
||||
return (
|
||||
page * BUF_NUMEL_PER_PAGE
|
||||
+ token_tile_id * (tile * HEAD_DIM)
|
||||
+ col_tile_id * (tile * tile)
|
||||
+ token_in_tile * tile
|
||||
+ col_in_tile
|
||||
)
|
||||
return page * BUF_NUMEL_PER_PAGE + slot * HEAD_DIM + cols
|
||||
|
||||
|
||||
def kpool_max_closed_pools(num_draft_tokens: int, pool_size: int) -> int:
|
||||
return (num_draft_tokens + pool_size - 1) // pool_size
|
||||
|
||||
@@ -59,6 +98,7 @@ def gather_index_k_scale_prefix_into(
|
||||
scale_out,
|
||||
PAGE_SIZE=pool.page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
@@ -76,6 +116,7 @@ def _gather_index_k_scale_prefix_into_kernel(
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0)
|
||||
@@ -85,7 +126,14 @@ def _gather_index_k_scale_prefix_into_kernel(
|
||||
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
mask = offs < HEAD_DIM
|
||||
src_k_offsets = page * BUF_NUMEL_PER_PAGE + token_offset_in_page * HEAD_DIM + offs
|
||||
src_k_offsets = _kpool_cache_k_offsets(
|
||||
page,
|
||||
token_offset_in_page,
|
||||
offs,
|
||||
BUF_NUMEL_PER_PAGE,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE_TILE,
|
||||
)
|
||||
dst_k_offsets = token_id * HEAD_DIM + offs
|
||||
k = tl.load(buf_u8_ptr + src_k_offsets, mask=mask)
|
||||
tl.store(k_out_ptr + dst_k_offsets, k, mask=mask)
|
||||
@@ -725,6 +773,7 @@ def kpool_softmax_rotate_write_cache(
|
||||
ape.stride(0),
|
||||
PAGE_SIZE=pool.page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=slot_k.shape[2],
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim,
|
||||
@@ -818,6 +867,7 @@ def kpool_decode_update_and_maybe_write_cache(
|
||||
REQ_POOL_SIZE=tail_k.shape[0],
|
||||
PAGE_SIZE=pool.page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=pool.index_kpool,
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=tail_k.shape[2],
|
||||
@@ -872,6 +922,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
POOL_SIZE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
@@ -947,10 +998,13 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
loc = tl.load(loc_ptr + row, mask=do_write, other=0)
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % PAGE_SIZE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE
|
||||
+ loc_token_offset_in_page * HEAD_DIM
|
||||
+ offs
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
BUF_NUMEL_PER_PAGE,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE_TILE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
@@ -1000,6 +1054,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_TABLE_COLS: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
@@ -1112,10 +1167,13 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
)
|
||||
loc_page_index = packed_page.to(tl.int64)
|
||||
loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE
|
||||
+ loc_token_offset_in_page * HEAD_DIM
|
||||
+ offs
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
BUF_NUMEL_PER_PAGE,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE_TILE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
@@ -1175,6 +1233,7 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
|
||||
TAIL_SIZE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
@@ -1222,8 +1281,13 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
|
||||
loc = tl.load(loc_ptr + row)
|
||||
loc_page_index = loc // SLOTS_PER_PAGE
|
||||
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE + loc_token_offset_in_page * HEAD_DIM + offs
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
BUF_NUMEL_PER_PAGE,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE_TILE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
@@ -1290,6 +1354,7 @@ def kpool_assemble_softmax_rotate_write_cache(
|
||||
tail_k.stride(1),
|
||||
ape.stride(0),
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=pool_size,
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
@@ -1537,6 +1602,7 @@ def _kpool_write_tail_and_maybe_compress_kernel(
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_EFFECTIVE_N: tl.constexpr,
|
||||
MAX_CLOSED_POOLS: tl.constexpr,
|
||||
@@ -1601,10 +1667,13 @@ def _kpool_write_tail_and_maybe_compress_kernel(
|
||||
loc = tl.load(write_loc_ptr + b * write_loc_stride_0 + p)
|
||||
loc_page_index = loc // SLOTS_PER_PAGE
|
||||
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
|
||||
out_k_offsets = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE
|
||||
+ loc_token_offset_in_page * HEAD_DIM
|
||||
+ offs
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
offs,
|
||||
BUF_NUMEL_PER_PAGE,
|
||||
HEAD_DIM,
|
||||
PRESHUFFLE_TILE,
|
||||
)
|
||||
out_s_offset = (
|
||||
loc_page_index * BUF_NUMEL_PER_PAGE // 4
|
||||
@@ -1694,6 +1763,7 @@ def kpool_write_tail_and_maybe_compress(
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
SLOTS_PER_PAGE=slots_per_page,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
S_OFFSET_NBYTES_IN_PAGE=slots_per_page * pool.index_head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_EFFECTIVE_N=effective_n_per_batch is not None,
|
||||
|
||||
Reference in New Issue
Block a user