[DSA] Chunk the kpool indexer MQA logits by query rows under a free-memory budget (#40854)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
2ee82fe158
commit
63d320c723
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -24,7 +25,10 @@ from sglang.srt.layers.attention.dsa.utils import (
|
||||
from sglang.srt.layers.attention.mqa_logits_utils import (
|
||||
MQA_LOGITS_BYTES_PER_ELEM,
|
||||
MQA_LOGITS_MAX_BYTES_ROCM,
|
||||
mqa_logits_budget_bytes,
|
||||
mqa_logits_row_bytes,
|
||||
mqa_logits_rows_per_chunk,
|
||||
mqa_logits_should_chunk,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import LayerNorm
|
||||
from sglang.srt.layers.utils import MultiPlatformOp
|
||||
@@ -57,8 +61,11 @@ from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils import capture_mode
|
||||
from sglang.srt.runtime_context import get_device, get_exec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
|
||||
@@ -69,6 +76,56 @@ def _should_fuse_kpool_topk(metadata: BaseIndexerMetadata) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _slice_rows(
|
||||
*, tensor: Optional[torch.Tensor], rows: slice
|
||||
) -> Optional[torch.Tensor]:
|
||||
return None if tensor is None else tensor[rows]
|
||||
|
||||
|
||||
def _mqa_logits_row_chunks(
|
||||
*, num_rows: int, num_cols: int, device: torch.device
|
||||
) -> Tuple[slice, ...]:
|
||||
# Real capture records a fixed launch count; breakable-graph replay also sets
|
||||
# get_is_capture_mode(), but its eager breaks run here and need the budget.
|
||||
if capture_mode.is_capture_mode or torch.cuda.is_current_stream_capturing():
|
||||
return (slice(0, num_rows),)
|
||||
device_index = device.index
|
||||
assert device_index is not None, "q_fp8 must be on an indexed CUDA device"
|
||||
# The logits kernel (DeepGEMM, or AITER on ROCm) allocates them outside every
|
||||
# pool sized by mem_fraction_static, so only the free-memory budget bounds them.
|
||||
need_chunk, budget_bytes = mqa_logits_should_chunk(
|
||||
num_rows=num_rows,
|
||||
num_cols=num_cols,
|
||||
get_budget_bytes=lambda: mqa_logits_budget_bytes(
|
||||
device_index=device_index, allow_sync=True
|
||||
),
|
||||
rocm=is_hip(),
|
||||
)
|
||||
rows_per_chunk = (
|
||||
mqa_logits_rows_per_chunk(
|
||||
num_rows=num_rows,
|
||||
row_bytes=mqa_logits_row_bytes(num_cols),
|
||||
budget_bytes=budget_bytes,
|
||||
)
|
||||
if need_chunk
|
||||
else None
|
||||
)
|
||||
if rows_per_chunk is None:
|
||||
return (slice(0, num_rows),)
|
||||
logger.debug(
|
||||
"kpool indexer chunks %d query rows x %d pooled cols into %d-row "
|
||||
"chunks (logits budget %d bytes)",
|
||||
num_rows,
|
||||
num_cols,
|
||||
rows_per_chunk,
|
||||
budget_bytes,
|
||||
)
|
||||
return tuple(
|
||||
slice(start, min(start + rows_per_chunk, num_rows))
|
||||
for start in range(0, num_rows, rows_per_chunk)
|
||||
)
|
||||
|
||||
|
||||
class IndexerKPool(MultiPlatformOp):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1007,18 +1064,81 @@ class IndexerKPool(MultiPlatformOp):
|
||||
)
|
||||
return topk_result
|
||||
|
||||
def _should_chunk_mqa_logits(
|
||||
self, num_q: int, num_k: int, device: torch.device
|
||||
) -> Tuple[bool, int]:
|
||||
if num_q * num_k < 8_000_000:
|
||||
return False, 0
|
||||
|
||||
free_mem, total_mem = torch.cuda.mem_get_info(device)
|
||||
bytes_per_elem = 4
|
||||
logits_bytes = num_q * num_k * bytes_per_elem
|
||||
|
||||
need_chunk = (logits_bytes * 2 > free_mem) or (logits_bytes > total_mem * 0.3)
|
||||
return need_chunk, free_mem
|
||||
def _kpool_topk_by_row_chunks(
|
||||
self,
|
||||
*,
|
||||
q_fp8: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
kv_fp8: Optional[Tuple[torch.Tensor, torch.Tensor]],
|
||||
logits_starts: Optional[torch.Tensor],
|
||||
logits_ends: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
seq_lens: Optional[torch.Tensor],
|
||||
page_table: Optional[torch.Tensor],
|
||||
page_table_row_index: Optional[torch.Tensor],
|
||||
topk_offsets: Optional[torch.Tensor],
|
||||
topk_row_starts: Optional[torch.Tensor],
|
||||
row_chunks: Tuple[slice, ...],
|
||||
out_rows: Optional[int],
|
||||
) -> torch.Tensor:
|
||||
# Each row's top-k reads only its own logits row, pooled length and
|
||||
# page-table row, so row chunks select the same pages as one pass.
|
||||
single_chunk = len(row_chunks) == 1
|
||||
topk_result = None
|
||||
for rows in row_chunks:
|
||||
if kv_fp8 is None:
|
||||
# No pooled keys yet: every row scores an empty key set.
|
||||
logits = torch.empty(
|
||||
(rows.stop - rows.start, 0),
|
||||
dtype=torch.float32,
|
||||
device=q_fp8.device,
|
||||
)
|
||||
else:
|
||||
assert logits_starts is not None
|
||||
k_fp8, k_scale = kv_fp8
|
||||
logits = self._fp8_mqa_logits(
|
||||
q_fp8=q_fp8[rows].contiguous(),
|
||||
k_fp8=k_fp8,
|
||||
k_scale=k_scale,
|
||||
weights=weights[rows].contiguous(),
|
||||
starts=logits_starts[rows],
|
||||
ends=logits_ends[rows],
|
||||
clean_logits=True,
|
||||
)
|
||||
topk_chunk = self._topk_from_kpool_logits(
|
||||
logits=logits,
|
||||
pool_lens=pool_lens[rows],
|
||||
seq_lens=_slice_rows(tensor=seq_lens, rows=rows),
|
||||
# Addressed by request-pool ID through page_table_row_index,
|
||||
# so never sliced; without a row index it is per-query.
|
||||
page_table=(
|
||||
page_table
|
||||
if page_table_row_index is not None
|
||||
else _slice_rows(tensor=page_table, rows=rows)
|
||||
),
|
||||
topk_offsets=_slice_rows(tensor=topk_offsets, rows=rows),
|
||||
row_starts=_slice_rows(tensor=topk_row_starts, rows=rows),
|
||||
out_rows=out_rows if single_chunk else None,
|
||||
page_table_row_index=_slice_rows(
|
||||
tensor=page_table_row_index, rows=rows
|
||||
),
|
||||
)
|
||||
if single_chunk:
|
||||
return topk_chunk
|
||||
# Free this chunk's logits before the next one allocates.
|
||||
del logits
|
||||
if topk_result is None:
|
||||
# Rows past the last chunk are padding, matching out_rows.
|
||||
num_out_rows = row_chunks[-1].stop if out_rows is None else out_rows
|
||||
topk_result = torch.full(
|
||||
(num_out_rows, topk_chunk.shape[1]),
|
||||
-1,
|
||||
dtype=topk_chunk.dtype,
|
||||
device=topk_chunk.device,
|
||||
)
|
||||
topk_result[rows] = topk_chunk
|
||||
assert topk_result is not None
|
||||
return topk_result
|
||||
|
||||
def _get_topk_ragged_kpool_plan(
|
||||
self,
|
||||
@@ -1050,6 +1170,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
f"plan has more real rows ({n_real}) than q_fp8 ({total_q})"
|
||||
)
|
||||
|
||||
kv_fp8 = None
|
||||
if total_k_rows > 0:
|
||||
k_u8 = plan.ragged_k_u8
|
||||
k_scale = plan.ragged_k_scale
|
||||
@@ -1064,17 +1185,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
scale_out=k_scale,
|
||||
)
|
||||
k_fp8 = k_u8.view(torch.float8_e4m3fn)
|
||||
logits = self._fp8_mqa_logits(
|
||||
q_fp8[:n_real].contiguous(),
|
||||
k_fp8.contiguous(),
|
||||
k_scale.contiguous(),
|
||||
weights[:n_real].contiguous(),
|
||||
ks_per_q,
|
||||
ke_per_q,
|
||||
clean_logits=True,
|
||||
)
|
||||
else:
|
||||
logits = torch.empty((n_real, 0), dtype=torch.float32, device=device)
|
||||
kv_fp8 = (k_fp8.contiguous(), k_scale.contiguous())
|
||||
|
||||
topk_method = metadata.topk_transform_method
|
||||
attn_metadata = metadata.attn_metadata
|
||||
@@ -1087,16 +1198,25 @@ class IndexerKPool(MultiPlatformOp):
|
||||
page_table_row_index_all = plan.ragged_paged_page_table_row_index
|
||||
elif topk_method == TopkTransformMethod.RAGGED:
|
||||
topk_offsets_all = attn_metadata.topk_indices_offset
|
||||
# The plan's table is req_to_token, indexed by request-pool ID.
|
||||
assert page_table_all is None or page_table_row_index_all is not None
|
||||
|
||||
return self._topk_from_kpool_logits(
|
||||
logits,
|
||||
pool_lens,
|
||||
return self._kpool_topk_by_row_chunks(
|
||||
q_fp8=q_fp8,
|
||||
weights=weights,
|
||||
kv_fp8=kv_fp8,
|
||||
logits_starts=ks_per_q,
|
||||
logits_ends=ke_per_q,
|
||||
pool_lens=pool_lens,
|
||||
seq_lens=seq_lens_expanded,
|
||||
page_table=page_table_all,
|
||||
topk_offsets=topk_offsets_all,
|
||||
row_starts=ks_per_q,
|
||||
out_rows=total_q,
|
||||
page_table_row_index=page_table_row_index_all,
|
||||
topk_offsets=topk_offsets_all,
|
||||
topk_row_starts=ks_per_q,
|
||||
row_chunks=_mqa_logits_row_chunks(
|
||||
num_rows=n_real, num_cols=total_k_rows, device=device
|
||||
),
|
||||
out_rows=total_q,
|
||||
)
|
||||
|
||||
def _get_topk_ragged_kpool(
|
||||
@@ -1297,19 +1417,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
and zero_starts_by_batch[i] is not None
|
||||
else torch.zeros((q_len,), dtype=torch.int32, device=q_fp8.device)
|
||||
)
|
||||
local_logits = self._fp8_mqa_logits(
|
||||
q_fp8[q_slice].contiguous(),
|
||||
k_fp8.contiguous(),
|
||||
k_scale.contiguous(),
|
||||
weights[q_slice].contiguous(),
|
||||
row_starts,
|
||||
local_pool_lens,
|
||||
clean_logits=True,
|
||||
)
|
||||
local_kv_fp8 = (k_fp8.contiguous(), k_scale.contiguous())
|
||||
else:
|
||||
local_logits = torch.empty(
|
||||
(q_len, 0), dtype=torch.float32, device=q_fp8.device
|
||||
)
|
||||
row_starts = None
|
||||
local_kv_fp8 = None
|
||||
|
||||
page_table_local = None
|
||||
topk_offsets_local = None
|
||||
@@ -1330,12 +1441,23 @@ class IndexerKPool(MultiPlatformOp):
|
||||
):
|
||||
topk_offsets_local = topk_offsets[q_slice]
|
||||
|
||||
local_topk = self._topk_from_kpool_logits(
|
||||
local_logits,
|
||||
local_pool_lens,
|
||||
# One request's logits still scale with its own context.
|
||||
local_topk = self._kpool_topk_by_row_chunks(
|
||||
q_fp8=q_fp8[q_slice],
|
||||
weights=weights[q_slice],
|
||||
kv_fp8=local_kv_fp8,
|
||||
logits_starts=row_starts,
|
||||
logits_ends=local_pool_lens,
|
||||
pool_lens=local_pool_lens,
|
||||
seq_lens=local_seqlens,
|
||||
page_table=page_table_local,
|
||||
page_table_row_index=None,
|
||||
topk_offsets=topk_offsets_local,
|
||||
topk_row_starts=None,
|
||||
row_chunks=_mqa_logits_row_chunks(
|
||||
num_rows=q_len, num_cols=pool_seq_len, device=q_fp8.device
|
||||
),
|
||||
out_rows=None,
|
||||
)
|
||||
|
||||
topk_result[q_slice] = local_topk
|
||||
|
||||
Reference in New Issue
Block a user