[Perf] [AMD] dsv4: FP4 indexer on aiter's row-group kernel for decode and target-verify (5/8) (#44006)
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
/// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]`
|
||||
/// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with
|
||||
/// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes.
|
||||
/// With kPage8 the same bytes of a page hold eight 8-slot pages in aiter's row-group layout
|
||||
/// (`pack_kv_cache(page_size=8)`): payload `[8, 4 (t % 4), 4 (chunk), 2 (t / 4), 16]`, scales `[8, 4, 8]`.
|
||||
|
||||
#pragma once
|
||||
|
||||
@@ -24,7 +26,14 @@ struct IndexKSplitParams {
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <bool kUsePDL, int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, typename PosT>
|
||||
template <
|
||||
bool kUsePDL,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
uint32_t kPageSize,
|
||||
uint32_t kRatio,
|
||||
bool kPage8,
|
||||
typename PosT>
|
||||
__global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel(
|
||||
const IndexKSplitParams params) {
|
||||
using namespace device;
|
||||
@@ -75,6 +84,23 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla
|
||||
|
||||
// slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing
|
||||
if (slot_id <= 0) return;
|
||||
if constexpr (kPage8) {
|
||||
constexpr int64_t kP8 = 8;
|
||||
const auto page8 = slot_id / kP8;
|
||||
const auto t = slot_id % kP8;
|
||||
// payload [t % 4][chunk][t / 4][16 B] and scales [chunk][t] of the slot's 8-slot page
|
||||
const auto payload_ptr = params.payload + page8 * (kP8 * kHeadDim / 2) + (t / 4) * 16;
|
||||
const auto chunk_row = (t % 4) * 4;
|
||||
payload_ptr[(chunk_row + lane / 16) * 32 + lane % 16] = static_cast<uint8_t>(packed.payload[0]);
|
||||
payload_ptr[(chunk_row + 2 + lane / 16) * 32 + lane % 16] = static_cast<uint8_t>(packed.payload[1]);
|
||||
if (lane % (kWarpThreads / 2) == 0) {
|
||||
const auto scale_ptr = params.scale + page8 * (kP8 * kHeadDim / 32) + t;
|
||||
const auto first = lane / (kWarpThreads / 2);
|
||||
scale_ptr[first * kP8] = static_cast<uint8_t>(packed.exponent[0]);
|
||||
scale_ptr[(2 + first) * kP8] = static_cast<uint8_t>(packed.exponent[1]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const auto page = slot_id / kPageSize;
|
||||
const auto slot = slot_id % kPageSize;
|
||||
|
||||
@@ -94,12 +120,13 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla
|
||||
}
|
||||
|
||||
/// \brief Host side of `flash_index_k_split_kernel`.
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kUsePDL>
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kPage8, bool kUsePDL>
|
||||
struct FlashIndexKSplitKernel {
|
||||
static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads;
|
||||
|
||||
template <typename PosT>
|
||||
static constexpr auto kernel = flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, PosT>;
|
||||
static constexpr auto kernel =
|
||||
flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, kPage8, PosT>;
|
||||
|
||||
/// `IndexKKernel::run_index_k`'s arguments, the cache replaced by
|
||||
/// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's
|
||||
|
||||
@@ -259,6 +259,20 @@ def _map_compact_selection_kernel(
|
||||
tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid)
|
||||
|
||||
|
||||
class CandidatePage8Table(NamedTuple):
|
||||
"""The candidate blocks of each row as a page-8 pool's row-group sequence: one row per
|
||||
sequence, its blocks one 8-slot page each, the valid columns a prefix."""
|
||||
|
||||
# int32 [rows, topk_blocks]: the kept ids, the newest (partial) block last, -1 padded
|
||||
ids: torch.Tensor
|
||||
# int32 [rows, topk_blocks]: the 8-slot pool page of each id, 0 padded
|
||||
block_tables: torch.Tensor
|
||||
# int32 [rows]: the reachable columns of the compact row
|
||||
compact_lens: torch.Tensor
|
||||
# int32 [rows + 1]: arange, one row a sequence
|
||||
query_start_loc: torch.Tensor
|
||||
|
||||
|
||||
class CandidateBlocks(NamedTuple):
|
||||
"""What the candidate-source layer publishes for the decode rows of a step."""
|
||||
|
||||
@@ -270,14 +284,26 @@ class CandidateBlocks(NamedTuple):
|
||||
compact_page_table: torch.Tensor
|
||||
compact_page_size: int
|
||||
block_size: int
|
||||
# on a page8 pool, built once and shared by the consumer layers
|
||||
page8: Optional[CandidatePage8Table] = None
|
||||
|
||||
|
||||
def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks:
|
||||
"""The rows rows of a per-request publication."""
|
||||
ids = candidates.ids[rows]
|
||||
page8 = candidates.page8
|
||||
if page8 is not None:
|
||||
page8 = CandidatePage8Table(
|
||||
page8.ids[rows],
|
||||
page8.block_tables[rows],
|
||||
page8.compact_lens[rows],
|
||||
page8.query_start_loc[: ids.shape[0] + 1],
|
||||
)
|
||||
return candidates._replace(
|
||||
ids=candidates.ids[rows],
|
||||
ids=ids,
|
||||
compact_lens=candidates.compact_lens[rows],
|
||||
compact_page_table=candidates.compact_page_table[rows],
|
||||
page8=page8,
|
||||
)
|
||||
|
||||
|
||||
@@ -289,6 +315,7 @@ def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks:
|
||||
ids=torch.cat([p.ids for p in pieces]),
|
||||
compact_lens=torch.cat([p.compact_lens for p in pieces]),
|
||||
compact_page_table=torch.cat([p.compact_page_table for p in pieces]),
|
||||
page8=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -389,18 +416,219 @@ def topk_within_candidate_blocks_hip(
|
||||
"""Level two for a consumer layer: the top-k of logits inside the published candidate
|
||||
blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with
|
||||
sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context."""
|
||||
rows, width = logits.shape
|
||||
seq_lens = seq_lens.to(torch.int32).contiguous()
|
||||
compact = gather_candidate_blocks(
|
||||
logits, seq_lens, candidates.ids, block_size=candidates.block_size
|
||||
)
|
||||
topk_compact_candidate_rows(
|
||||
compact,
|
||||
candidates.compact_lens,
|
||||
candidates.ids,
|
||||
candidates,
|
||||
seq_lens,
|
||||
page_table=page_table,
|
||||
page_size=page_size,
|
||||
page_indices=page_indices,
|
||||
raw_indices=raw_indices,
|
||||
sort_output=sort_output,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _candidate_page8_table_kernel(
|
||||
ids_ptr,
|
||||
seq_lens_ptr,
|
||||
page_table_ptr,
|
||||
out_ids_ptr,
|
||||
pages_ptr,
|
||||
lens_ptr,
|
||||
query_start_loc_ptr,
|
||||
ids_stride,
|
||||
pt_stride,
|
||||
out_stride,
|
||||
n_pages,
|
||||
K,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCKS_PER_PAGE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""One program per row (kept ids a -1 padded prefix): moves the newest, partial block
|
||||
last so the reachable columns are a prefix, and writes the physical page table and that
|
||||
exact length."""
|
||||
row = tl.program_id(0)
|
||||
j = tl.arange(0, BLOCK)
|
||||
in_k = j < K
|
||||
base = ids_ptr + row * ids_stride
|
||||
ids = tl.load(base + j, mask=in_k, other=-1)
|
||||
live = ids >= 0
|
||||
kept = tl.sum(live.to(tl.int32), axis=0)
|
||||
length = tl.load(seq_lens_ptr + row)
|
||||
newest = tl.maximum(length - 1, 0) // BLOCK_SIZE
|
||||
at = tl.max(tl.where(live & (ids == newest), j, -1), axis=0)
|
||||
last = kept - 1
|
||||
swap = at >= 0
|
||||
at_last = tl.sum(tl.where(j == last, ids, 0), axis=0)
|
||||
ids = tl.where(swap & (j == last), newest, tl.where(swap & (j == at), at_last, ids))
|
||||
span = tl.where(
|
||||
swap,
|
||||
tl.minimum(tl.maximum(length - newest * BLOCK_SIZE, 0), BLOCK_SIZE),
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
tl.store(lens_ptr + row, tl.where(kept > 0, last * BLOCK_SIZE + span, 0))
|
||||
blk = tl.where(live, ids, 0)
|
||||
page = tl.load(
|
||||
page_table_ptr
|
||||
+ row * pt_stride
|
||||
+ tl.minimum(blk // BLOCKS_PER_PAGE, n_pages - 1),
|
||||
mask=live,
|
||||
other=0,
|
||||
)
|
||||
pages = page * BLOCKS_PER_PAGE + blk % BLOCKS_PER_PAGE
|
||||
tl.store(out_ids_ptr + row * out_stride + j, ids, mask=in_k)
|
||||
tl.store(pages_ptr + row * out_stride + j, tl.where(live, pages, 0), mask=in_k)
|
||||
tl.store(query_start_loc_ptr + row + 1, row + 1)
|
||||
if row == 0:
|
||||
tl.store(query_start_loc_ptr, 0)
|
||||
|
||||
|
||||
def candidate_page8_table(
|
||||
ids: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
*,
|
||||
block_size: int,
|
||||
page_size: int,
|
||||
) -> CandidatePage8Table:
|
||||
"""The CandidatePage8Table of each row's kept block ids."""
|
||||
rows, k = ids.shape
|
||||
assert block_size == 8 and page_size % block_size == 0, (block_size, page_size)
|
||||
assert ids.dtype == torch.int32 and ids.stride(1) == 1
|
||||
assert page_table.dtype == torch.int32 and page_table.stride(1) == 1
|
||||
out_ids = torch.empty((rows, k), dtype=torch.int32, device=ids.device)
|
||||
pages = torch.empty_like(out_ids)
|
||||
lens = torch.empty(rows, dtype=torch.int32, device=ids.device)
|
||||
query_start_loc = torch.empty(rows + 1, dtype=torch.int32, device=ids.device)
|
||||
if not rows:
|
||||
query_start_loc.zero_()
|
||||
else:
|
||||
_candidate_page8_table_kernel[(rows,)](
|
||||
ids,
|
||||
seq_lens.to(torch.int32).contiguous(),
|
||||
page_table,
|
||||
out_ids,
|
||||
pages,
|
||||
lens,
|
||||
query_start_loc,
|
||||
ids.stride(0),
|
||||
page_table.stride(0),
|
||||
out_ids.stride(0),
|
||||
page_table.shape[1],
|
||||
k,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCKS_PER_PAGE=page_size // block_size,
|
||||
BLOCK=triton.next_power_of_2(k),
|
||||
)
|
||||
return CandidatePage8Table(out_ids, pages, lens, query_start_loc)
|
||||
|
||||
|
||||
def with_candidate_page8_table(
|
||||
candidates: CandidateBlocks,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
) -> CandidateBlocks:
|
||||
"""candidates with their page8 table attached."""
|
||||
return candidates._replace(
|
||||
page8=candidate_page8_table(
|
||||
candidates.ids,
|
||||
seq_lens,
|
||||
page_table,
|
||||
block_size=candidates.block_size,
|
||||
page_size=page_size,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def topk_within_candidate_blocks_page8(
|
||||
*,
|
||||
q_fp4: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
k_payload: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
candidates: CandidateBlocks,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
page_size: int,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
is_decode: bool = True,
|
||||
) -> None:
|
||||
"""topk_within_candidate_blocks_hip on a page8 index-K pool, scoring only the candidate
|
||||
blocks."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
rowgroup_paged_mqa_logits,
|
||||
)
|
||||
|
||||
block_size = candidates.block_size
|
||||
table = candidates.page8 or candidate_page8_table(
|
||||
candidates.ids, seq_lens, page_table, block_size=block_size, page_size=page_size
|
||||
)
|
||||
compact = rowgroup_paged_mqa_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights,
|
||||
weight_scale=1.0,
|
||||
block_tables=table.block_tables,
|
||||
query_start_loc=table.query_start_loc,
|
||||
row_ends=table.compact_lens,
|
||||
max_query_len=1,
|
||||
max_seq_len=candidates.ids.shape[1] * block_size,
|
||||
page8=True,
|
||||
pages_per_block=1,
|
||||
is_decode=is_decode,
|
||||
)
|
||||
topk_compact_candidate_rows(
|
||||
compact,
|
||||
table.compact_lens,
|
||||
table.ids,
|
||||
candidates,
|
||||
seq_lens,
|
||||
page_table=page_table,
|
||||
page_size=page_size,
|
||||
page_indices=page_indices,
|
||||
raw_indices=raw_indices,
|
||||
)
|
||||
|
||||
|
||||
def topk_compact_candidate_rows(
|
||||
compact: torch.Tensor,
|
||||
compact_lens: torch.Tensor,
|
||||
ids: torch.Tensor,
|
||||
candidates: CandidateBlocks,
|
||||
seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_table: torch.Tensor,
|
||||
page_size: int,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
sort_output: bool = False,
|
||||
) -> None:
|
||||
"""The top-k of each compact candidate row (column j * block_size + t is position
|
||||
ids[j] * block_size + t, valid up to compact_lens), mapped back to pool slots as
|
||||
topk_within_candidate_blocks_hip writes them."""
|
||||
rows = compact.shape[0]
|
||||
topk = page_indices.shape[1]
|
||||
block_size = candidates.block_size
|
||||
seq_lens = seq_lens.to(torch.int32).contiguous()
|
||||
compact = gather_candidate_blocks(
|
||||
logits, seq_lens, candidates.ids, block_size=block_size
|
||||
)
|
||||
assert compact.shape[1] <= candidates.compact_page_size
|
||||
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
|
||||
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=compact.device)
|
||||
topk_transform_paged_hip(
|
||||
compact,
|
||||
candidates.compact_lens,
|
||||
compact_lens,
|
||||
candidates.compact_page_table,
|
||||
compact_pos,
|
||||
candidates.compact_page_size,
|
||||
@@ -415,12 +643,12 @@ def topk_within_candidate_blocks_hip(
|
||||
assert not sort_output or topk & (topk - 1) == 0, topk
|
||||
_map_compact_selection_kernel[(rows,)](
|
||||
compact_pos,
|
||||
candidates.ids,
|
||||
ids,
|
||||
seq_lens,
|
||||
page_table,
|
||||
page_indices,
|
||||
raw_indices if write_raw else page_indices,
|
||||
candidates.ids.stride(0),
|
||||
ids.stride(0),
|
||||
page_table.stride(0),
|
||||
page_indices.stride(0),
|
||||
page_table.shape[1],
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import os
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -46,6 +48,15 @@ _PREFILL_BASE_CTA_TARGET = 1024
|
||||
LOW_RATIO_PAGE_TABLE_BUCKET = 64
|
||||
# AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len].
|
||||
_DECODE_CTA_INFO_WIDTH = 4
|
||||
# Row-group decode: a request's verify rows share each key load from this many requests up;
|
||||
# below it each row scoring as its own sequence is faster (gfx950, H = 32, 8K..64K context).
|
||||
_ROWGROUP_MIN_GROUPED_SEQS = 4
|
||||
# DeepSeek-V4 C4 indexer (H = 64, 64-slot pages): a target-verify scores on the row-group
|
||||
# kernel, a request's rows one sequence, from this many rows up (4-row DSpark: 16 requests,
|
||||
# 7-row: 10); below it the prefill kernel's schedule is as fast or faster. MI355X, in a
|
||||
# DSpark decode step: 7 rows x 16 requests at 131K 1.36x, 4 x 32 per DP rank at 32K 1.11x,
|
||||
# 4 x 8 at 64K 0.88x (bench_dsv4_c4_indexer_rowgroup_hip.py for the kernel alone).
|
||||
C4_ROWGROUP_MIN_VERIFY_ROWS = 64
|
||||
|
||||
# Budget for the pooled prefill logits block, in MiB. Rows are split to fit it
|
||||
# (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint
|
||||
@@ -68,6 +79,17 @@ class FP4DecodeWorkspace(NamedTuple):
|
||||
schedule_scratch: torch.Tensor
|
||||
|
||||
|
||||
class FP4RowgroupDecodeWorkspace(NamedTuple):
|
||||
"""Decode workspace of the schedule-free row-group kernel."""
|
||||
|
||||
guarded_page_table: torch.Tensor
|
||||
max_seq_len: int
|
||||
# sequence b's rows are query_start_loc[b] .. [b + 1] - 1, rows_per_seq each,
|
||||
# all on page-table row query_start_loc[b]
|
||||
query_start_loc: torch.Tensor
|
||||
rows_per_seq: int
|
||||
|
||||
|
||||
class FP4PrefillWorkspace(NamedTuple):
|
||||
guarded_page_table: torch.Tensor
|
||||
row_to_batch: torch.Tensor
|
||||
@@ -81,6 +103,16 @@ class FP4PrefillWorkspace(NamedTuple):
|
||||
schedule_buffers: Optional[PrefillScheduleBuffers] = None
|
||||
|
||||
|
||||
class FP4RowgroupPrefillWorkspace(NamedTuple):
|
||||
"""Prefill workspace of the row-group kernel, no schedule: each request's padded
|
||||
page-table row, and per scored row range the sequences' query_start_loc, built on first
|
||||
use and shared by the step's layers."""
|
||||
|
||||
request_page_table: torch.Tensor
|
||||
max_seq_len: int
|
||||
query_start_locs: Dict[Tuple[int, int], torch.Tensor]
|
||||
|
||||
|
||||
class FP4KWriteMetadata(NamedTuple):
|
||||
positions: torch.Tensor
|
||||
slots: torch.Tensor
|
||||
@@ -155,17 +187,32 @@ def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int:
|
||||
)
|
||||
|
||||
|
||||
def rowgroup_table_columns(
|
||||
max_context_len: int, ratio: int, page_table_bucket: int = 4
|
||||
) -> int:
|
||||
"""Block-table columns of every row-group launch at ``ratio``: one width for the
|
||||
server's life. AITER compiles the row-group kernel per block-table width and stride,
|
||||
and its bounded compile cache unloads an evicted variant that a captured CUDA graph
|
||||
still launches; widths that track the context would also compile on the request path."""
|
||||
compressed_len = -(-max_context_len // ratio)
|
||||
pages = -(-compressed_len // _KV_BLOCK_SIZE)
|
||||
return _guarded_pages(pages, page_table_bucket) + 4
|
||||
|
||||
|
||||
def _guard_page_table(
|
||||
page_table: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
page_table_bucket: int = 4,
|
||||
columns: Optional[int] = None,
|
||||
):
|
||||
"""Pad page tables for 256-token scheduling and one-chunk lookahead."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
|
||||
pad_page_table,
|
||||
)
|
||||
|
||||
return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket)
|
||||
return pad_page_table(
|
||||
page_table, out=out, page_table_bucket=page_table_bucket, columns=columns
|
||||
)
|
||||
|
||||
|
||||
def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int:
|
||||
@@ -244,6 +291,89 @@ def prepare_fp4_decode_workspace(
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def aiter_has_fp4_rowgroup() -> bool:
|
||||
"""Whether the installed AITER ships the row-group FP4 MQA logits kernel (#6145)."""
|
||||
try:
|
||||
from aiter.ops.flydsl import make_fp4_mqa_plan # noqa: F401
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def prepare_fp4_rowgroup_decode_workspace(
|
||||
page_table: torch.Tensor,
|
||||
rows_per_request: int,
|
||||
page_table_bucket: int = 4,
|
||||
table_columns: Optional[int] = None,
|
||||
min_grouped_seqs: int = _ROWGROUP_MIN_GROUPED_SEQS,
|
||||
) -> FP4RowgroupDecodeWorkspace:
|
||||
"""The decode workspace of the row-group logits kernel: the padded page table and the
|
||||
rows' sequences, no schedule. ``rows_per_request`` consecutive rows belong to one
|
||||
request (one page-table row, non-decreasing bounds); they score as one sequence once
|
||||
there are ``min_grouped_seqs`` requests, else every row is its own. The table has at
|
||||
least ``table_columns`` columns (``rowgroup_table_columns``). Capture-safe."""
|
||||
guarded, max_seq_len = _guard_page_table(
|
||||
page_table, page_table_bucket=page_table_bucket, columns=table_columns
|
||||
)
|
||||
rows = guarded.shape[0]
|
||||
grouped = (
|
||||
rows_per_request > 1
|
||||
and rows % rows_per_request == 0
|
||||
and rows // rows_per_request >= min_grouped_seqs
|
||||
)
|
||||
rows_per_seq = rows_per_request if grouped else 1
|
||||
query_start_loc = torch.arange(
|
||||
0, rows + 1, rows_per_seq, dtype=torch.int32, device=guarded.device
|
||||
)
|
||||
return FP4RowgroupDecodeWorkspace(
|
||||
guarded, max_seq_len, query_start_loc, rows_per_seq
|
||||
)
|
||||
|
||||
|
||||
def _pinned_int32(values: List[int], device: torch.device) -> torch.Tensor:
|
||||
return (
|
||||
torch.tensor(values, dtype=torch.int32)
|
||||
.pin_memory()
|
||||
.to(device, non_blocking=True)
|
||||
)
|
||||
|
||||
|
||||
def prepare_fp4_rowgroup_prefill_workspace(
|
||||
page_table: torch.Tensor,
|
||||
extend_lens: List[int],
|
||||
page_table_bucket: int = 4,
|
||||
table_columns: Optional[int] = None,
|
||||
) -> FP4RowgroupPrefillWorkspace:
|
||||
"""The row-group prefill workspace of a step whose requests own extend_lens consecutive
|
||||
page-table rows each, the request table at least ``table_columns`` columns
|
||||
(``rowgroup_table_columns``)."""
|
||||
num_rows = page_table.shape[0]
|
||||
starts = itertools.accumulate([0, *extend_lens[:-1]])
|
||||
first_rows = [min(s, num_rows - 1) for s in starts]
|
||||
request_page_table, max_seq_len = _guard_page_table(
|
||||
page_table.index_select(0, _pinned_int32(first_rows, page_table.device)),
|
||||
page_table_bucket=page_table_bucket,
|
||||
columns=table_columns,
|
||||
)
|
||||
return FP4RowgroupPrefillWorkspace(request_page_table, max_seq_len, {})
|
||||
|
||||
|
||||
def rowgroup_prefill_query_start_loc(
|
||||
workspace: FP4RowgroupPrefillWorkspace, rows: slice, rows_per_request: List[int]
|
||||
) -> torch.Tensor:
|
||||
"""query_start_loc of the consecutive requests rows_per_request scored as rows."""
|
||||
key = (rows.start, rows.stop)
|
||||
query_start_loc = workspace.query_start_locs.get(key)
|
||||
if query_start_loc is None:
|
||||
query_start_loc = _pinned_int32(
|
||||
[0, *itertools.accumulate(rows_per_request)],
|
||||
workspace.request_page_table.device,
|
||||
)
|
||||
workspace.query_start_locs[key] = query_start_loc
|
||||
return query_start_loc
|
||||
|
||||
|
||||
def prepare_fp4_prefill_workspace(
|
||||
page_table: torch.Tensor,
|
||||
c4_seq_lens: torch.Tensor,
|
||||
@@ -309,6 +439,103 @@ def prepare_fp4_prefill_workspace(
|
||||
return workspace
|
||||
|
||||
|
||||
def _rowgroup_decode_logits(
|
||||
*,
|
||||
q_fp4: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
k_payload: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
weight_scale: float,
|
||||
workspace: FP4RowgroupDecodeWorkspace,
|
||||
row_ends: torch.Tensor,
|
||||
page8: bool = False,
|
||||
is_decode: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Decode logits from AITER's row-group kernel over a row-group workspace."""
|
||||
q = workspace.rows_per_seq
|
||||
return rowgroup_paged_mqa_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights,
|
||||
weight_scale=weight_scale,
|
||||
block_tables=workspace.guarded_page_table[::q],
|
||||
query_start_loc=workspace.query_start_loc,
|
||||
row_ends=row_ends,
|
||||
max_query_len=q,
|
||||
max_seq_len=workspace.max_seq_len,
|
||||
page8=page8,
|
||||
pages_per_block=8 if page8 else 1,
|
||||
is_decode=is_decode,
|
||||
)
|
||||
|
||||
|
||||
def rowgroup_paged_mqa_logits(
|
||||
*,
|
||||
q_fp4: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
k_payload: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
weight_scale: float,
|
||||
block_tables: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
row_ends: torch.Tensor,
|
||||
max_query_len: int,
|
||||
max_seq_len: int,
|
||||
page8: bool,
|
||||
pages_per_block: int = 1,
|
||||
is_decode: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""[rows, max_seq_len] fp32 logits from AITER's row-group kernel: sequence b's rows
|
||||
query_start_loc[b] .. [b + 1] - 1 score the keys of block_tables[b] up to their own
|
||||
row_ends. An entry names a 64-slot page of the pool, or with page8 pages_per_block of
|
||||
the pool's 8-slot pages (8: the 64-slot page table as is; 1: an 8-slot page each, the
|
||||
candidate table). Past a row's bound the logits are left unwritten; the length-aware
|
||||
top-k never reads them."""
|
||||
from aiter.ops.flydsl import flydsl_pa_mqa_logits_fp4, make_fp4_mqa_plan
|
||||
|
||||
num_rows, heads = q_fp4.shape[0], q_fp4.shape[-2]
|
||||
if page8:
|
||||
kv, ks = index_k_page8_views(k_payload, k_scale)
|
||||
page_size = 8
|
||||
else:
|
||||
assert pages_per_block == 1, "a 64-slot page entry names one page"
|
||||
kv, ks = k_payload.view(torch.uint8), k_scale
|
||||
page_size = _KV_BLOCK_SIZE
|
||||
plan = make_fp4_mqa_plan(
|
||||
num_seqs=query_start_loc.shape[0] - 1,
|
||||
max_qlen=max_query_len,
|
||||
num_rows=num_rows,
|
||||
heads=heads,
|
||||
page_size=page_size,
|
||||
max_seq_len=max_seq_len,
|
||||
pages_per_block=pages_per_block,
|
||||
)
|
||||
logits = _alloc_logits(num_rows, max_seq_len, q_fp4.device, is_decode)
|
||||
flydsl_pa_mqa_logits_fp4(
|
||||
q_fp4.view(torch.uint8).reshape(num_rows, 1, heads, _HEAD_DIM // 2),
|
||||
q_scale.reshape(num_rows, 1, *_Q_SCALE_SHAPE),
|
||||
kv,
|
||||
ks,
|
||||
block_tables,
|
||||
weights,
|
||||
None,
|
||||
max_seq_len,
|
||||
weight_scale=weight_scale,
|
||||
kv_block_size=page_size,
|
||||
out=logits,
|
||||
row_ends=_as_int32_1d(row_ends),
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=max_query_len,
|
||||
pages_per_block=pages_per_block,
|
||||
plan=plan,
|
||||
)
|
||||
return logits
|
||||
|
||||
|
||||
def aiter_fp4_paged_mqa_logits(
|
||||
*,
|
||||
q_fp4: torch.Tensor,
|
||||
@@ -320,11 +547,17 @@ def aiter_fp4_paged_mqa_logits(
|
||||
c4_seq_lens: torch.Tensor,
|
||||
weight_scale: float,
|
||||
is_decode: bool,
|
||||
decode_workspace: Optional[FP4DecodeWorkspace] = None,
|
||||
decode_workspace: Optional[
|
||||
Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
|
||||
] = None,
|
||||
prefill_workspace: Optional[FP4PrefillWorkspace] = None,
|
||||
page_table_bucket: int = 4,
|
||||
page8: bool = False,
|
||||
rowgroup_table_columns: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel."""
|
||||
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel. A page8 pool
|
||||
is read by the row-group kernel only: without a row-group decode workspace every row
|
||||
scores as its own sequence, on a table of ``rowgroup_table_columns`` columns."""
|
||||
from aiter.ops.flydsl import (
|
||||
flydsl_pa_mqa_logits_fp4,
|
||||
flydsl_pa_mqa_logits_fp4_prefill,
|
||||
@@ -336,7 +569,42 @@ def aiter_fp4_paged_mqa_logits(
|
||||
# A workspace is bound to one row count. DP padding or truncated activations
|
||||
# can leave it stale, in which case fall back to building the schedule here.
|
||||
if workspace is not None and workspace.guarded_page_table.shape[0] != num_tokens:
|
||||
workspace = None
|
||||
if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace):
|
||||
# Stay on the row-group kernel, capture-safe and on the same compiled table
|
||||
# width, scoring every row as its own sequence.
|
||||
workspace = prepare_fp4_rowgroup_decode_workspace(
|
||||
page_table,
|
||||
1,
|
||||
page_table_bucket=page_table_bucket,
|
||||
table_columns=workspace.guarded_page_table.shape[1],
|
||||
)
|
||||
else:
|
||||
workspace = None
|
||||
rowgroup_ws = (
|
||||
workspace
|
||||
if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace)
|
||||
else None
|
||||
)
|
||||
if page8 and rowgroup_ws is None:
|
||||
rowgroup_ws = prepare_fp4_rowgroup_decode_workspace(
|
||||
page_table,
|
||||
1,
|
||||
page_table_bucket=page_table_bucket,
|
||||
table_columns=rowgroup_table_columns,
|
||||
)
|
||||
if rowgroup_ws is not None:
|
||||
return _rowgroup_decode_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights,
|
||||
weight_scale=weight_scale,
|
||||
workspace=rowgroup_ws,
|
||||
row_ends=c4_seq_lens,
|
||||
page8=page8,
|
||||
is_decode=is_decode,
|
||||
)
|
||||
# Built on the fallback path below; kept in scope so the schedule scratch
|
||||
# outlives the logits kernel that reads it.
|
||||
fallback_schedule = None
|
||||
@@ -555,6 +823,7 @@ def _store_fp4_index_k_split_kernel(
|
||||
loc,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
PAGE8: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0)
|
||||
cache_loc = tl.load(loc + token_id)
|
||||
@@ -566,6 +835,23 @@ def _store_fp4_index_k_split_kernel(
|
||||
chunk = offsets // 16
|
||||
byte = offsets - chunk * 16
|
||||
k = tl.load(k_fp4 + token_id * BLOCK + offsets)
|
||||
sf = tl.load(k_sf + token_id)
|
||||
sf_offsets = tl.arange(0, 4)
|
||||
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
|
||||
if PAGE8:
|
||||
# aiter's row-group 8-slot page: payload [t % 4][chunk][t // 4][16 B], scales [chunk][t]
|
||||
page8 = cache_loc // 8
|
||||
t = cache_loc % 8
|
||||
tl.store(
|
||||
payload
|
||||
+ page8 * (8 * BLOCK)
|
||||
+ ((t % 4) * 4 + chunk) * 32
|
||||
+ (t // 4) * 16
|
||||
+ byte,
|
||||
k,
|
||||
)
|
||||
tl.store(scale + page8 * 32 + sf_offsets * 8 + t, sf_bytes)
|
||||
return
|
||||
tl.store(
|
||||
payload
|
||||
+ page * (4 * page_size * 16)
|
||||
@@ -576,9 +862,6 @@ def _store_fp4_index_k_split_kernel(
|
||||
)
|
||||
# scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI)
|
||||
shuffled = (page_offset % 16) * 4 + page_offset // 16
|
||||
sf = tl.load(k_sf + token_id)
|
||||
sf_offsets = tl.arange(0, 4)
|
||||
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
|
||||
tl.store(
|
||||
scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes
|
||||
)
|
||||
@@ -592,10 +875,12 @@ def store_fp4_index_k_cache_split(
|
||||
*,
|
||||
page_size: int,
|
||||
rne: bool = False,
|
||||
page8: bool = False,
|
||||
) -> None:
|
||||
"""Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot
|
||||
loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16],
|
||||
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed)."""
|
||||
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed); with page8
|
||||
each page holds eight 8-slot pages of aiter's row-group layout (pack_kv_cache)."""
|
||||
assert input.shape[-1] == _HEAD_DIM
|
||||
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
|
||||
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
|
||||
@@ -612,17 +897,43 @@ def store_fp4_index_k_cache_split(
|
||||
loc,
|
||||
page_size,
|
||||
BLOCK=64,
|
||||
PAGE8=page8,
|
||||
)
|
||||
|
||||
|
||||
def index_k_page8_views(
|
||||
payload: torch.Tensor, scale: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A page8 pool's buffers as aiter's 8-slot pages: (payload uint8 [pages * 8, 512],
|
||||
scale uint8 [pages * 8, 32]); 8-slot page p covers slots 8p .. 8p + 7."""
|
||||
pages8 = payload.shape[0] * (payload.shape[3] // 8)
|
||||
return (
|
||||
payload.view(torch.uint8).view(pages8, 8 * _HEAD_DIM // 2),
|
||||
scale.view(pages8, 8 * _HEAD_DIM // 32),
|
||||
)
|
||||
|
||||
|
||||
def read_fp4_index_k_split(
|
||||
payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
payload: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
page8: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales
|
||||
int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of
|
||||
quantize_fp4_indexer_tensor."""
|
||||
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
|
||||
slots = slots.to(torch.int64)
|
||||
if page8:
|
||||
kv8, ks8 = index_k_page8_views(payload, scale)
|
||||
page, t = slots // 8, slots % 8
|
||||
kv8 = kv8.view(-1, 4, 4, 2, 16) # [page8, t % 4, chunk, t // 4, 16]
|
||||
rows = kv8[page, t % 4, :, t // 4, :].reshape(-1, 64).view(torch.int8)
|
||||
sf = ks8.view(-1, 4, 8)[page, :, t].to(torch.int32) # [n, 4]
|
||||
packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24)
|
||||
return rows, packed
|
||||
page, off = slots // page_size, slots % page_size
|
||||
rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16]
|
||||
rows = rows.reshape(-1, 64).view(torch.int8)
|
||||
|
||||
@@ -21,9 +21,11 @@ if TYPE_CHECKING:
|
||||
|
||||
@cache_once
|
||||
def _jit_index_k_split_module(
|
||||
head_dim: int, rope_dim: int, page_size: int, ratio: int
|
||||
head_dim: int, rope_dim: int, page_size: int, ratio: int, page8: bool
|
||||
) -> Module:
|
||||
args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl())
|
||||
args = make_cpp_args(
|
||||
head_dim, rope_dim, page_size, ratio, page8, is_arch_support_pdl()
|
||||
)
|
||||
return load_jit(
|
||||
make_name("index_k_rope_pack_split"),
|
||||
*args,
|
||||
@@ -43,16 +45,18 @@ def index_k_norm_rope_pack_store_split(
|
||||
scale: torch.Tensor,
|
||||
*,
|
||||
ratio: int,
|
||||
page8: bool = False,
|
||||
) -> None:
|
||||
"""`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split
|
||||
FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4,
|
||||
page_size] uint8); the bytes equal store_fp4_index_k_cache_split's."""
|
||||
page_size] uint8), with page8 each page as eight row-group 8-slot pages; the bytes
|
||||
equal store_fp4_index_k_cache_split's."""
|
||||
head_dim = input.shape[-1]
|
||||
page_size = payload.shape[3]
|
||||
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
|
||||
assert scale.shape[1:] == (1, 4, page_size), scale.shape
|
||||
_jit_index_k_split_module(
|
||||
head_dim, freqs_cis.shape[-1], page_size, ratio
|
||||
head_dim, freqs_cis.shape[-1], page_size, ratio, page8
|
||||
).index_k_split(
|
||||
input,
|
||||
norm_weight,
|
||||
|
||||
@@ -198,21 +198,24 @@ def pad_page_table(
|
||||
page_table: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
page_table_bucket: int = 4,
|
||||
columns: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, int]:
|
||||
"""Pad a page table for 256-token scheduling in a single dispatch.
|
||||
|
||||
Replaces the ``new_zeros`` + masked ``copy_`` pair: the kernel writes every
|
||||
output element, so the destination never needs pre-zeroing.
|
||||
output element, so the destination never needs pre-zeroing. ``columns`` widens
|
||||
the destination to at least that many columns; the returned logits width stays
|
||||
the table's own.
|
||||
"""
|
||||
page_table = _as_int32_2d(page_table)
|
||||
rows, logical_width, padded_width = padded_page_table_shape(
|
||||
page_table, page_table_bucket
|
||||
)
|
||||
w_dst = max(padded_width + 4, columns or 0)
|
||||
if out is None:
|
||||
out = page_table.new_empty((rows, padded_width + 4))
|
||||
out = page_table.new_empty((rows, w_dst))
|
||||
else:
|
||||
assert out.shape == (rows, padded_width + 4), f"{out.shape=} {rows=}"
|
||||
w_dst = padded_width + 4
|
||||
assert out.shape == (rows, w_dst), f"{out.shape=} {rows=}"
|
||||
if rows:
|
||||
_pad_page_table_kernel[(rows, triton.cdiv(w_dst, _PT_BLOCK))](
|
||||
page_table,
|
||||
|
||||
@@ -1688,6 +1688,16 @@ class Envs:
|
||||
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
|
||||
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
|
||||
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
|
||||
# V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group
|
||||
# FP4 MQA logits kernel, a request's verify rows sharing each key load.
|
||||
SGLANG_HIP_FP4_INDEXER_ROWGROUP = EnvBool(_default_hip)
|
||||
# DeepSeek-V4 C4 decode / target-verify indexer: the same row-group kernel (aiter >= #6145),
|
||||
# its workspace built in the graph without a schedule; a request's verify rows share each
|
||||
# key load.
|
||||
SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP = EnvBool(False)
|
||||
# V4.1 ratio-1 index-K pool in aiter's row-group page-8 layout (8 consecutive 8-slot pages per
|
||||
# 64-slot page), so a candidate consumer layer scores only its published blocks.
|
||||
SGLANG_HIP_DSV41_INDEX_K_PAGE8 = EnvBool(False)
|
||||
|
||||
# cache, GEMM, and distributed
|
||||
SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True)
|
||||
|
||||
@@ -65,6 +65,7 @@ from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4AttnBackend,
|
||||
LateLayerTail,
|
||||
_as_int_list,
|
||||
_prefill_graph_max_seq_len,
|
||||
_tail_rows,
|
||||
)
|
||||
@@ -120,6 +121,8 @@ if TYPE_CHECKING:
|
||||
FP4DecodeWorkspace,
|
||||
FP4KWriteMetadata,
|
||||
FP4PrefillWorkspace,
|
||||
FP4RowgroupDecodeWorkspace,
|
||||
FP4RowgroupPrefillWorkspace,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -684,15 +687,17 @@ class DSV4Metadata:
|
||||
# Per-step scratch for the TP-padded query heads (models/deepseek_v4.py).
|
||||
q_pad_buffer: Optional[torch.Tensor] = None
|
||||
# low-ratio FlyDSL indexer workspaces by ratio; address-pinned like the c4 ones, rebuilt in place
|
||||
fp4_low_ratio_decode_workspaces: Dict[int, FP4DecodeWorkspace] = field(
|
||||
default_factory=dict, repr=False
|
||||
)
|
||||
fp4_low_ratio_prefill_workspaces: Dict[int, FP4PrefillWorkspace] = field(
|
||||
default_factory=dict, repr=False
|
||||
)
|
||||
fp4_low_ratio_decode_workspaces: Dict[
|
||||
int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
|
||||
] = field(default_factory=dict, repr=False)
|
||||
fp4_low_ratio_prefill_workspaces: Dict[
|
||||
int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace]
|
||||
] = field(default_factory=dict, repr=False)
|
||||
# Captured kernels bind these FP4 indexer buffers by address; absent from copy_
|
||||
# so addresses stay pinned across replays (the builders refresh contents).
|
||||
fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False)
|
||||
fp4_decode_workspace: Optional[
|
||||
Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
|
||||
] = field(default=None, repr=False)
|
||||
fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field(
|
||||
default=None, repr=False
|
||||
)
|
||||
@@ -1015,6 +1020,36 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
|
||||
self._fp4_graph_row_limit = MAX_FUSED_ROWS
|
||||
# DSv4 C4 decode / target-verify indexer on AITER's row-group kernel: no schedule,
|
||||
# its workspace built in the graph, a request's verify rows sharing each key load.
|
||||
# Every launch reads a table of one width (its compiled variant) for the server's life.
|
||||
self._fp4_c4_rowgroup_columns: Optional[int] = None
|
||||
if (
|
||||
self.enable_deepseek_v4_fp4_indexer
|
||||
and self.speculative_num_steps == 0
|
||||
and self.has_c4
|
||||
and envs.SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP.get()
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
aiter_has_fp4_rowgroup,
|
||||
rowgroup_table_columns,
|
||||
)
|
||||
|
||||
if aiter_has_fp4_rowgroup():
|
||||
self._fp4_c4_rowgroup_columns = rowgroup_table_columns(
|
||||
max(self.max_context_len, self.MAX_SEQ_LEN_FOR_CAPTURE), 4
|
||||
)
|
||||
logger.info(
|
||||
"DSv4 C4 indexer: decode / target-verify on the row-group FP4 "
|
||||
"MQA logits kernel (%d table columns)",
|
||||
self._fp4_c4_rowgroup_columns,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP is set but the installed AITER "
|
||||
"has no row-group FP4 MQA logits kernel (#6145); keeping the "
|
||||
"schedule-based C4 indexer."
|
||||
)
|
||||
self.speculative_step_id = speculative_step_id
|
||||
self.forward_metadata: Union[
|
||||
DSV4Metadata,
|
||||
@@ -1728,13 +1763,29 @@ class DeepseekV4HipRadixBackend(
|
||||
if not low_ratio_decode_rows_are_identity(
|
||||
self, forward_batch, ratio
|
||||
)
|
||||
}
|
||||
},
|
||||
max_context_len=self.max_context_len,
|
||||
num_requests=forward_batch.batch_size,
|
||||
page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(),
|
||||
)
|
||||
)
|
||||
|
||||
# Decode's schedule builder is capture-safe (the workspace pins its scratch);
|
||||
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace.
|
||||
if self._fp4_workspaces_enabled(metadata) and (
|
||||
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. The
|
||||
# row-group workspace has no schedule, so target-verify builds it here too.
|
||||
rowgroup_rows = self._fp4_c4_rowgroup_rows(metadata, forward_batch)
|
||||
if rowgroup_rows is not None:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
prepare_fp4_rowgroup_decode_workspace,
|
||||
)
|
||||
|
||||
metadata.fp4_decode_workspace = prepare_fp4_rowgroup_decode_workspace(
|
||||
metadata.indexer_metadata.page_table,
|
||||
rowgroup_rows,
|
||||
table_columns=self._fp4_c4_rowgroup_columns,
|
||||
min_grouped_seqs=1,
|
||||
)
|
||||
elif self._fp4_workspaces_enabled(metadata) and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
@@ -1758,6 +1809,40 @@ class DeepseekV4HipRadixBackend(
|
||||
and metadata.c4_compress_metadata is not None
|
||||
)
|
||||
|
||||
def _fp4_c4_rowgroup_rows(
|
||||
self, metadata, forward_batch: ForwardBatch
|
||||
) -> Optional[int]:
|
||||
"""Rows a request scores as one sequence on the row-group kernel this step: 1 for
|
||||
decode, the draft rows for a target-verify of at least C4_ROWGROUP_MIN_VERIFY_ROWS
|
||||
rows (below it the prefill kernel's schedule spreads a few requests' keys wider).
|
||||
None keeps the schedule kernels, as does a verify whose logits outgrow the prefill
|
||||
budget (the row-group launch is unchunked).
|
||||
"""
|
||||
if self._fp4_c4_rowgroup_columns is None or not self._fp4_workspaces_enabled(
|
||||
metadata
|
||||
):
|
||||
return None
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_decode():
|
||||
return 1
|
||||
if not mode.is_target_verify():
|
||||
return None
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
C4_ROWGROUP_MIN_VERIFY_ROWS,
|
||||
logits_rows_per_chunk,
|
||||
)
|
||||
|
||||
page_table = metadata.indexer_metadata.page_table
|
||||
rows, num_draft = page_table.shape[0], self.target_verify_num_draft_tokens
|
||||
if (
|
||||
num_draft < 2
|
||||
or rows != num_draft * forward_batch.batch_size
|
||||
or rows < C4_ROWGROUP_MIN_VERIFY_ROWS
|
||||
or rows > logits_rows_per_chunk(page_table)
|
||||
):
|
||||
return None
|
||||
return num_draft
|
||||
|
||||
def _refresh_fp4_prefill_workspace(
|
||||
self, forward_batch: ForwardBatch, metadata: Optional[DSV4Metadata] = None
|
||||
) -> None:
|
||||
@@ -1775,6 +1860,13 @@ class DeepseekV4HipRadixBackend(
|
||||
refresh_low_ratio_prefill_workspaces(
|
||||
metadata.low_ratio_indexer_metadata_by_ratio(),
|
||||
metadata.fp4_low_ratio_prefill_workspaces,
|
||||
extend_lens=(
|
||||
metadata.late_layer_tail.extend_seq_lens_cpu
|
||||
if metadata.late_layer_tail is not None
|
||||
else _as_int_list(forward_batch.extend_seq_lens_cpu)
|
||||
),
|
||||
max_context_len=self.max_context_len,
|
||||
page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(),
|
||||
)
|
||||
)
|
||||
if not self._fp4_workspaces_enabled(metadata):
|
||||
@@ -1785,6 +1877,9 @@ class DeepseekV4HipRadixBackend(
|
||||
ForwardMode.TARGET_VERIFY,
|
||||
):
|
||||
return
|
||||
if self._fp4_c4_rowgroup_rows(metadata, forward_batch) is not None:
|
||||
# scored on the row-group kernel, its workspace built with the metadata
|
||||
return
|
||||
if (
|
||||
get_parallel().attn_cp_size != 1
|
||||
or getattr(forward_batch, "tbo_children", None)
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.kernels.ops.attention.dsv4 import (
|
||||
topk_transform_paged_v2,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
FP4RowgroupDecodeWorkspace,
|
||||
aiter_fp4_paged_mqa_logits,
|
||||
aiter_q_indexer_fp4,
|
||||
logits_rows_per_chunk,
|
||||
@@ -1018,6 +1019,14 @@ class C4IndexerBackendMixin:
|
||||
elif use_aiter_fp4:
|
||||
q_fp4, q_scale = q
|
||||
is_decode = forward_batch.forward_mode.is_decode()
|
||||
if isinstance(
|
||||
getattr(metadata, "fp4_decode_workspace", None),
|
||||
FP4RowgroupDecodeWorkspace,
|
||||
) and (is_decode or forward_batch.forward_mode.is_target_verify()):
|
||||
# The backend built a row-group workspace for this step: target-verify
|
||||
# rows score like decode rows, one launch over the whole batch, a
|
||||
# request's rows sharing each key load.
|
||||
is_decode = True
|
||||
# Hoisted: these await this layer's KV transfer, which every chunk
|
||||
# would otherwise re-await.
|
||||
k_payload = token_to_kv_pool.get_index_k_fp4_payload_buffer(
|
||||
|
||||
@@ -3,8 +3,19 @@ payload / scale index-K pools, then the top-k v2 transform -- the DeepGEMM path'
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
||||
import logging
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Dict,
|
||||
FrozenSet,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -17,12 +28,17 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
|
||||
slice_candidate_blocks,
|
||||
topk_transform_paged_hip,
|
||||
topk_within_candidate_blocks_hip,
|
||||
topk_within_candidate_blocks_page8,
|
||||
with_candidate_page8_table,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
FP4DecodeWorkspace,
|
||||
FP4PrefillWorkspace,
|
||||
FP4RowgroupDecodeWorkspace,
|
||||
FP4RowgroupPrefillWorkspace,
|
||||
aiter_fp4_paged_mqa_logits,
|
||||
aiter_has_fp4_rowgroup,
|
||||
index_q_rope_pack_flydsl,
|
||||
index_q_rope_pack_weights_flydsl,
|
||||
indexer_head_weights,
|
||||
@@ -30,11 +46,17 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
pack_fp4_query_flydsl,
|
||||
prepare_fp4_decode_workspace,
|
||||
prepare_fp4_prefill_workspace,
|
||||
prepare_fp4_rowgroup_decode_workspace,
|
||||
prepare_fp4_rowgroup_prefill_workspace,
|
||||
rowgroup_paged_mqa_logits,
|
||||
rowgroup_prefill_query_start_loc,
|
||||
rowgroup_table_columns,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import (
|
||||
index_k_norm_rope_pack_store_split,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_as_int_list,
|
||||
)
|
||||
@@ -47,6 +69,61 @@ if TYPE_CHECKING:
|
||||
# fp32 logits scored per candidate-block chunk stay under this many bytes
|
||||
_CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _fp4_rowgroup_enabled() -> bool:
|
||||
"""SGLANG_HIP_FP4_INDEXER_ROWGROUP, off when the installed AITER has no row-group
|
||||
kernel (ROCm/aiter#6145)."""
|
||||
if not envs.SGLANG_HIP_FP4_INDEXER_ROWGROUP.get():
|
||||
return False
|
||||
if not aiter_has_fp4_rowgroup():
|
||||
logger.warning(
|
||||
"SGLANG_HIP_FP4_INDEXER_ROWGROUP is set but the installed AITER has no "
|
||||
"row-group FP4 MQA logits kernel (#6145); keeping the schedule kernels."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _score_requests_page8(
|
||||
*,
|
||||
q_fp4: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
k_payload: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
workspace: FP4RowgroupPrefillWorkspace,
|
||||
rows: slice,
|
||||
req_lo: int,
|
||||
rows_per_request: List[int],
|
||||
row_ends: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Full-context logits of a page8 pool for rows, consecutive requests from req_lo on with
|
||||
rows_per_request rows each; a request's rows share every key load."""
|
||||
assert sum(rows_per_request) == q_fp4.shape[0], (rows_per_request, q_fp4.shape)
|
||||
return rowgroup_paged_mqa_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights,
|
||||
weight_scale=1.0,
|
||||
block_tables=workspace.request_page_table[
|
||||
req_lo : req_lo + len(rows_per_request)
|
||||
],
|
||||
query_start_loc=rowgroup_prefill_query_start_loc(
|
||||
workspace, rows, rows_per_request
|
||||
),
|
||||
row_ends=row_ends,
|
||||
max_query_len=max(rows_per_request),
|
||||
max_seq_len=workspace.max_seq_len,
|
||||
page8=True,
|
||||
pages_per_block=8,
|
||||
is_decode=False,
|
||||
)
|
||||
|
||||
|
||||
def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool:
|
||||
"""Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch."""
|
||||
@@ -114,33 +191,74 @@ def _indexer_inputs(layer, x, q_lora, pos):
|
||||
|
||||
def build_low_ratio_decode_workspaces(
|
||||
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
|
||||
) -> Dict[int, FP4DecodeWorkspace]:
|
||||
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace."""
|
||||
return {
|
||||
ratio: prepare_fp4_decode_workspace(
|
||||
meta.page_table,
|
||||
meta.compressed_seq_lens,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
*,
|
||||
max_context_len: int,
|
||||
num_requests: Optional[int] = None,
|
||||
page8_ratios: FrozenSet[int] = frozenset(),
|
||||
) -> Dict[int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]]:
|
||||
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace.
|
||||
With the row-group kernel there is no schedule; a request's rows (``num_requests``
|
||||
requests of equal row counts, consecutive: decode 1, target-verify its draft rows)
|
||||
score as one sequence, on a table as wide as ``max_context_len`` needs. A page8 pool
|
||||
is read by the row-group kernel only."""
|
||||
rowgroup = _fp4_rowgroup_enabled()
|
||||
workspaces = {}
|
||||
for ratio, meta in metadata_by_ratio.items():
|
||||
if not (rowgroup or ratio in page8_ratios):
|
||||
workspaces[ratio] = prepare_fp4_decode_workspace(
|
||||
meta.page_table,
|
||||
meta.compressed_seq_lens,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
)
|
||||
continue
|
||||
rows = meta.page_table.shape[0]
|
||||
rows_per_request = (
|
||||
rows // num_requests if num_requests and rows % num_requests == 0 else 1
|
||||
)
|
||||
for ratio, meta in metadata_by_ratio.items()
|
||||
}
|
||||
workspaces[ratio] = prepare_fp4_rowgroup_decode_workspace(
|
||||
meta.page_table,
|
||||
rows_per_request,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
table_columns=rowgroup_table_columns(
|
||||
max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
|
||||
),
|
||||
)
|
||||
return workspaces
|
||||
|
||||
|
||||
def refresh_low_ratio_prefill_workspaces(
|
||||
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
|
||||
previous: Optional[Dict[int, FP4PrefillWorkspace]],
|
||||
) -> Dict[int, FP4PrefillWorkspace]:
|
||||
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace."""
|
||||
*,
|
||||
extend_lens: Optional[List[int]],
|
||||
max_context_len: int,
|
||||
page8_ratios: FrozenSet[int] = frozenset(),
|
||||
) -> Dict[int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace]]:
|
||||
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace. A page8 pool
|
||||
scores on the row-group kernel, which takes no schedule, on a table as wide as
|
||||
``max_context_len`` needs (without extend_lens, the indexer builds its workspace
|
||||
itself)."""
|
||||
previous = previous or {}
|
||||
return {
|
||||
ratio: prepare_fp4_prefill_workspace(
|
||||
workspaces = {}
|
||||
for ratio, meta in metadata_by_ratio.items():
|
||||
if ratio in page8_ratios:
|
||||
if extend_lens is not None:
|
||||
workspaces[ratio] = prepare_fp4_rowgroup_prefill_workspace(
|
||||
meta.page_table,
|
||||
extend_lens,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
table_columns=rowgroup_table_columns(
|
||||
max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
|
||||
),
|
||||
)
|
||||
continue
|
||||
workspaces[ratio] = prepare_fp4_prefill_workspace(
|
||||
meta.page_table,
|
||||
meta.compressed_seq_lens,
|
||||
workspace=previous.get(ratio),
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
)
|
||||
for ratio, meta in metadata_by_ratio.items()
|
||||
}
|
||||
return workspaces
|
||||
|
||||
|
||||
def low_ratio_identity_skip_enabled(
|
||||
@@ -277,11 +395,37 @@ def low_ratio_index_topk_hip_decode(
|
||||
two_level = False
|
||||
|
||||
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
|
||||
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
|
||||
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
|
||||
page8 = pool.low_ratio_index_k_page8(layer.layer_id)
|
||||
if two_level and indexer.uses_candidates:
|
||||
candidates = backend.candidate_masks
|
||||
assert (
|
||||
isinstance(candidates, CandidateBlocks)
|
||||
and candidates.ids.shape[0] == q_fp4.shape[0]
|
||||
and candidates.block_size == indexer.candidate_block_size
|
||||
), "candidate blocks missing for decode"
|
||||
if page8:
|
||||
topk_within_candidate_blocks_page8(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
weights=weights,
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
candidates=candidates,
|
||||
seq_lens=indexer_metadata.compressed_seq_lens,
|
||||
page_table=indexer_metadata.page_table,
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=core.sparse_page_indices(ratio),
|
||||
raw_indices=core.sparse_raw_indices(ratio),
|
||||
is_decode=True,
|
||||
)
|
||||
return
|
||||
logits = aiter_fp4_paged_mqa_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id),
|
||||
k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id),
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights,
|
||||
page_table=indexer_metadata.page_table,
|
||||
c4_seq_lens=indexer_metadata.compressed_seq_lens,
|
||||
@@ -289,15 +433,13 @@ def low_ratio_index_topk_hip_decode(
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
is_decode=True,
|
||||
decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio),
|
||||
page8=page8,
|
||||
rowgroup_table_columns=rowgroup_table_columns(
|
||||
backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
|
||||
),
|
||||
)
|
||||
# level one is bounded on device by the compressed lengths: a captured step cannot read them back
|
||||
if two_level and indexer.uses_candidates:
|
||||
candidates = backend.candidate_masks
|
||||
assert (
|
||||
isinstance(candidates, CandidateBlocks)
|
||||
and candidates.ids.shape[0] == logits.shape[0]
|
||||
and candidates.block_size == indexer.candidate_block_size
|
||||
), "candidate blocks missing for decode"
|
||||
topk_within_candidate_blocks_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
@@ -309,12 +451,20 @@ def low_ratio_index_topk_hip_decode(
|
||||
)
|
||||
return
|
||||
if two_level and indexer.is_candidate_source:
|
||||
backend.candidate_masks = select_candidate_blocks_hip(
|
||||
candidates = select_candidate_blocks_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
topk_blocks=indexer.candidate_topk_blocks,
|
||||
block_size=indexer.candidate_block_size,
|
||||
)
|
||||
if page8:
|
||||
candidates = with_candidate_page8_table(
|
||||
candidates,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
indexer_metadata.page_table,
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
)
|
||||
backend.candidate_masks = candidates
|
||||
topk_transform_paged_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
@@ -426,8 +576,36 @@ def low_ratio_index_topk_hip_extend(
|
||||
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
|
||||
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
|
||||
prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio)
|
||||
page8 = pool.low_ratio_index_k_page8(layer.layer_id)
|
||||
if page8 and not (
|
||||
isinstance(prefill_workspace, FP4RowgroupPrefillWorkspace)
|
||||
and prefill_workspace.request_page_table.shape[0] == len(extend_lens_cpu)
|
||||
):
|
||||
prefill_workspace = prepare_fp4_rowgroup_prefill_workspace(
|
||||
indexer_metadata.page_table,
|
||||
extend_lens_cpu,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
table_columns=rowgroup_table_columns(
|
||||
backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
|
||||
),
|
||||
)
|
||||
|
||||
def score_rows(rows: slice) -> torch.Tensor:
|
||||
def score_rows(
|
||||
rows: slice, req_lo: int, rows_per_request: List[int]
|
||||
) -> torch.Tensor:
|
||||
if page8:
|
||||
return _score_requests_page8(
|
||||
q_fp4=q_fp4[rows],
|
||||
q_scale=q_scale[rows],
|
||||
weights=weights[rows],
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
workspace=prefill_workspace,
|
||||
rows=rows,
|
||||
req_lo=req_lo,
|
||||
rows_per_request=rows_per_request,
|
||||
row_ends=indexer_metadata.compressed_seq_lens[rows],
|
||||
)
|
||||
return aiter_fp4_paged_mqa_logits(
|
||||
q_fp4=q_fp4[rows],
|
||||
q_scale=q_scale[rows],
|
||||
@@ -442,16 +620,58 @@ def low_ratio_index_topk_hip_extend(
|
||||
prefill_workspace=prefill_workspace if rows.start == 0 else None,
|
||||
)
|
||||
|
||||
def select_rows_page8_consumer(rows: slice, req_lo, rows_per_request, consume_rows):
|
||||
"""A consumer on a page8 pool scores only a request's candidate blocks."""
|
||||
|
||||
def full_logits(r: slice, b: int) -> torch.Tensor:
|
||||
g = slice(rows.start + r.start, rows.start + r.stop)
|
||||
return score_rows(g, req_lo + b, [r.stop - r.start])
|
||||
|
||||
def within(r: slice, candidates, rows_page, rows_raw) -> None:
|
||||
g = slice(rows.start + r.start, rows.start + r.stop)
|
||||
topk_within_candidate_blocks_page8(
|
||||
q_fp4=q_fp4[g],
|
||||
q_scale=q_scale[g],
|
||||
weights=weights[g],
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
candidates=candidates,
|
||||
seq_lens=compress_lens[g],
|
||||
page_table=indexer_metadata.page_table[g],
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=rows_page,
|
||||
raw_indices=rows_raw,
|
||||
is_decode=False,
|
||||
)
|
||||
|
||||
_consume_candidates_per_request(
|
||||
extend_lens_cpu=rows_per_request,
|
||||
consume=consume_rows,
|
||||
compress_lens=compress_lens[rows],
|
||||
page_table=indexer_metadata.page_table[rows],
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=page_indices[rows],
|
||||
raw_indices=raw_indices[rows] if raw_indices is not None else None,
|
||||
full_logits=full_logits,
|
||||
within=within,
|
||||
)
|
||||
|
||||
def select_rows(
|
||||
rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish
|
||||
):
|
||||
rows_per_request = (
|
||||
[rows.stop - rows.start]
|
||||
if req_hi == req_lo + 1
|
||||
else extend_lens_cpu[req_lo:req_hi]
|
||||
)
|
||||
if page8 and consume_rows is not None:
|
||||
select_rows_page8_consumer(rows, req_lo, rows_per_request, consume_rows)
|
||||
return
|
||||
_select_topk_extend_hip(
|
||||
indexer=indexer,
|
||||
logits=score_rows(rows),
|
||||
logits=score_rows(rows, req_lo, rows_per_request),
|
||||
lc_per_req=lc_per_req[req_lo:req_hi],
|
||||
extend_lens_cpu=[rows.stop - rows.start]
|
||||
if req_hi == req_lo + 1
|
||||
else extend_lens_cpu[req_lo:req_hi],
|
||||
extend_lens_cpu=rows_per_request,
|
||||
is_identity=group_is_identity,
|
||||
compress_lens=compress_lens[rows],
|
||||
page_table=indexer_metadata.page_table[rows],
|
||||
@@ -503,6 +723,19 @@ def low_ratio_index_topk_hip_extend(
|
||||
if publish is not None:
|
||||
publish.append(cat_candidate_blocks(pieces))
|
||||
if publish is not None:
|
||||
if page8:
|
||||
assert len(publish) == len(extend_lens_cpu), (len(publish), extend_lens_cpu)
|
||||
tok = 0
|
||||
for b, candidates in enumerate(publish):
|
||||
r = slice(tok, tok + extend_lens_cpu[b])
|
||||
tok = r.stop
|
||||
if candidates is not None:
|
||||
publish[b] = with_candidate_page8_table(
|
||||
candidates,
|
||||
compress_lens[r],
|
||||
indexer_metadata.page_table[r],
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
)
|
||||
backend.candidate_masks = publish
|
||||
|
||||
|
||||
@@ -551,33 +784,27 @@ def _select_topk_extend_hip(
|
||||
)
|
||||
publish.append(cat_candidate_blocks(pieces))
|
||||
if consume is not None:
|
||||
tok_start = 0
|
||||
for b, t_len in enumerate(extend_lens_cpu):
|
||||
rows = slice(tok_start, tok_start + t_len)
|
||||
tok_start += t_len
|
||||
if t_len == 0:
|
||||
continue
|
||||
rows_page = page_indices[rows]
|
||||
rows_raw = raw_indices[rows] if raw_indices is not None else None
|
||||
if consume[b] is None:
|
||||
topk_transform_paged_hip(
|
||||
logits[rows],
|
||||
compress_lens[rows].contiguous(),
|
||||
page_table[rows],
|
||||
rows_page,
|
||||
page_size,
|
||||
rows_raw,
|
||||
)
|
||||
else:
|
||||
_consume_candidates_per_request(
|
||||
extend_lens_cpu=extend_lens_cpu,
|
||||
consume=consume,
|
||||
compress_lens=compress_lens,
|
||||
page_table=page_table,
|
||||
page_size=page_size,
|
||||
page_indices=page_indices,
|
||||
raw_indices=raw_indices,
|
||||
full_logits=lambda rows, b: logits[rows],
|
||||
within=lambda rows, candidates, rows_page, rows_raw: (
|
||||
topk_within_candidate_blocks_hip(
|
||||
logits[rows],
|
||||
compress_lens[rows],
|
||||
consume[b],
|
||||
candidates,
|
||||
page_table=page_table[rows],
|
||||
page_size=page_size,
|
||||
page_indices=rows_page,
|
||||
raw_indices=rows_raw,
|
||||
)
|
||||
),
|
||||
)
|
||||
return
|
||||
topk_transform_paged_hip(
|
||||
logits,
|
||||
@@ -589,6 +816,43 @@ def _select_topk_extend_hip(
|
||||
)
|
||||
|
||||
|
||||
def _consume_candidates_per_request(
|
||||
*,
|
||||
extend_lens_cpu: List[int],
|
||||
consume: List[Optional[CandidateBlocks]],
|
||||
compress_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
page_size: int,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
full_logits: Callable[[slice, int], torch.Tensor],
|
||||
within: Callable[
|
||||
[slice, CandidateBlocks, torch.Tensor, Optional[torch.Tensor]], None
|
||||
],
|
||||
) -> None:
|
||||
"""A consumer layer's rows, request by request: within(rows, candidates, ...) selects inside
|
||||
the published blocks; a request with none runs the paged top-k of full_logits(rows, b)."""
|
||||
tok_start = 0
|
||||
for b, t_len in enumerate(extend_lens_cpu):
|
||||
rows = slice(tok_start, tok_start + t_len)
|
||||
tok_start += t_len
|
||||
if t_len == 0:
|
||||
continue
|
||||
rows_page = page_indices[rows]
|
||||
rows_raw = raw_indices[rows] if raw_indices is not None else None
|
||||
if consume[b] is None:
|
||||
topk_transform_paged_hip(
|
||||
full_logits(rows, b),
|
||||
compress_lens[rows].contiguous(),
|
||||
page_table[rows],
|
||||
rows_page,
|
||||
page_size,
|
||||
rows_raw,
|
||||
)
|
||||
else:
|
||||
within(rows, consume[b], rows_page, rows_raw)
|
||||
|
||||
|
||||
def _request_groups(
|
||||
extend_lens_cpu: List[int], rows_per_chunk: int
|
||||
) -> List[Tuple[int, int, int, int]]:
|
||||
@@ -620,4 +884,5 @@ def store_index_k_split(
|
||||
pool.get_index_k_fp4_payload_buffer(layer_id),
|
||||
pool.get_index_k_fp4_scale_buffer(layer_id),
|
||||
ratio=layer.compress_ratio,
|
||||
page8=pool.low_ratio_index_k_page8(layer_id),
|
||||
)
|
||||
|
||||
@@ -2,7 +2,16 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import List, Literal, NamedTuple, Optional, Sequence, Tuple, Union
|
||||
from typing import (
|
||||
FrozenSet,
|
||||
List,
|
||||
Literal,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Sequence,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
import torch
|
||||
|
||||
@@ -557,6 +566,10 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
self.uses_aiter_fp4_layout = _is_hip and self.use_fp4_indexer
|
||||
# Low-ratio pools round to nearest even; c4 keeps threshold rounding.
|
||||
self.index_k_rne = False
|
||||
# aiter's row-group page-8 layout inside each page (pack_kv_cache): page size and shapes
|
||||
# are unchanged but the bytes are not, so PD peers must use the same layout and a
|
||||
# persisted cache written under the other one must be cleared before switching
|
||||
self.index_k_page8 = False
|
||||
|
||||
self._create_buffer()
|
||||
|
||||
@@ -702,6 +715,7 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
loc,
|
||||
page_size=self.page_size,
|
||||
rne=self.index_k_rne,
|
||||
page8=self.index_k_page8,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
store_fp4_index_k_cache,
|
||||
@@ -1533,6 +1547,19 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
kv_layout=self.compressed_kv_layout(ratio),
|
||||
)
|
||||
|
||||
index_k_page8 = _is_hip and envs.SGLANG_HIP_DSV41_INDEX_K_PAGE8.get()
|
||||
if index_k_page8:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
aiter_has_fp4_rowgroup,
|
||||
)
|
||||
|
||||
if not aiter_has_fp4_rowgroup():
|
||||
logger.warning(
|
||||
"SGLANG_HIP_DSV41_INDEX_K_PAGE8 is set but the installed AITER has "
|
||||
"no row-group FP4 MQA logits kernel (#6145); keeping the page-64 "
|
||||
"index-K pool."
|
||||
)
|
||||
index_k_page8 = False
|
||||
self.index_pools: dict[int, DeepSeekV4IndexerPool] = {
|
||||
ratio: self._make_indexer_pool(
|
||||
config.indexer_size,
|
||||
@@ -1559,6 +1586,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
device,
|
||||
enable_memory_saver,
|
||||
force_fp4=True,
|
||||
# only the candidate consumers (ratio 1) score a subset of blocks
|
||||
index_k_page8=ratio == 1 and index_k_page8,
|
||||
)
|
||||
|
||||
# HiCache and hardware backends still read these per-ratio attributes.
|
||||
@@ -1616,6 +1645,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
enable_memory_saver: bool,
|
||||
force_fp4: bool = False,
|
||||
global_page_size: Optional[int] = None,
|
||||
index_k_page8: bool = False,
|
||||
) -> DeepSeekV4IndexerPool:
|
||||
"""Build the c4 lightning-indexer K pool (packed CUDA layout).
|
||||
Overridden by :class:`DSV4NPUTokenToKVPool` to swap in the
|
||||
@@ -1636,6 +1666,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
)
|
||||
# The dsv41 low-ratio indexer rounds to nearest even (reference rounding).
|
||||
pool.index_k_rne = True
|
||||
pool.index_k_page8 = index_k_page8
|
||||
return pool
|
||||
return DeepSeekV4IndexerPool(
|
||||
size,
|
||||
@@ -1998,6 +2029,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
compress_ratio, _, _ = self.layer_mapping[layer_id]
|
||||
return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout
|
||||
|
||||
def low_ratio_index_k_page8(self, layer_id: int) -> bool:
|
||||
"""Whether the layer's split index-K pool lays each page out as row-group 8-slot pages."""
|
||||
compress_ratio, _, _ = self.layer_mapping[layer_id]
|
||||
return self._indexer_pool(compress_ratio).index_k_page8
|
||||
|
||||
def low_ratio_index_k_page8_ratios(self) -> FrozenSet[int]:
|
||||
"""The compression ratios whose index-K pool is in the page-8 layout."""
|
||||
return frozenset(
|
||||
ratio for ratio, pool in self.index_pools.items() if pool.index_k_page8
|
||||
)
|
||||
|
||||
def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
self.wait_layer_transfer(layer_id)
|
||||
compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id]
|
||||
|
||||
Reference in New Issue
Block a user