[Perf] [AMD] dsv4: FP4 indexer on aiter's row-group kernel for decode and target-verify (5/8) (#44006)

Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
kk
2026-10-16 02:16:06 -07:00
committed by GitHub
co-authored by Duyi-Wang
parent 9fde8bcc46
commit 6efc96abf9
13 changed files with 2148 additions and 90 deletions
@@ -1,6 +1,8 @@
/// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]`
/// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with
/// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes.
/// With kPage8 the same bytes of a page hold eight 8-slot pages in aiter's row-group layout
/// (`pack_kv_cache(page_size=8)`): payload `[8, 4 (t % 4), 4 (chunk), 2 (t / 4), 16]`, scales `[8, 4, 8]`.
#pragma once
@@ -24,7 +26,14 @@ struct IndexKSplitParams {
float eps;
};
template <bool kUsePDL, int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, typename PosT>
template <
bool kUsePDL,
int64_t kHeadDim,
int64_t kRopeDim,
uint32_t kPageSize,
uint32_t kRatio,
bool kPage8,
typename PosT>
__global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel(
const IndexKSplitParams params) {
using namespace device;
@@ -75,6 +84,23 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla
// slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing
if (slot_id <= 0) return;
if constexpr (kPage8) {
constexpr int64_t kP8 = 8;
const auto page8 = slot_id / kP8;
const auto t = slot_id % kP8;
// payload [t % 4][chunk][t / 4][16 B] and scales [chunk][t] of the slot's 8-slot page
const auto payload_ptr = params.payload + page8 * (kP8 * kHeadDim / 2) + (t / 4) * 16;
const auto chunk_row = (t % 4) * 4;
payload_ptr[(chunk_row + lane / 16) * 32 + lane % 16] = static_cast<uint8_t>(packed.payload[0]);
payload_ptr[(chunk_row + 2 + lane / 16) * 32 + lane % 16] = static_cast<uint8_t>(packed.payload[1]);
if (lane % (kWarpThreads / 2) == 0) {
const auto scale_ptr = params.scale + page8 * (kP8 * kHeadDim / 32) + t;
const auto first = lane / (kWarpThreads / 2);
scale_ptr[first * kP8] = static_cast<uint8_t>(packed.exponent[0]);
scale_ptr[(2 + first) * kP8] = static_cast<uint8_t>(packed.exponent[1]);
}
return;
}
const auto page = slot_id / kPageSize;
const auto slot = slot_id % kPageSize;
@@ -94,12 +120,13 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla
}
/// \brief Host side of `flash_index_k_split_kernel`.
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kUsePDL>
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, uint32_t kRatio, bool kPage8, bool kUsePDL>
struct FlashIndexKSplitKernel {
static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads;
template <typename PosT>
static constexpr auto kernel = flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, PosT>;
static constexpr auto kernel =
flash_index_k_split_kernel<kUsePDL, kHeadDim, kRopeDim, kPageSize, kRatio, kPage8, PosT>;
/// `IndexKKernel::run_index_k`'s arguments, the cache replaced by
/// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's
@@ -259,6 +259,20 @@ def _map_compact_selection_kernel(
tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid)
class CandidatePage8Table(NamedTuple):
"""The candidate blocks of each row as a page-8 pool's row-group sequence: one row per
sequence, its blocks one 8-slot page each, the valid columns a prefix."""
# int32 [rows, topk_blocks]: the kept ids, the newest (partial) block last, -1 padded
ids: torch.Tensor
# int32 [rows, topk_blocks]: the 8-slot pool page of each id, 0 padded
block_tables: torch.Tensor
# int32 [rows]: the reachable columns of the compact row
compact_lens: torch.Tensor
# int32 [rows + 1]: arange, one row a sequence
query_start_loc: torch.Tensor
class CandidateBlocks(NamedTuple):
"""What the candidate-source layer publishes for the decode rows of a step."""
@@ -270,14 +284,26 @@ class CandidateBlocks(NamedTuple):
compact_page_table: torch.Tensor
compact_page_size: int
block_size: int
# on a page8 pool, built once and shared by the consumer layers
page8: Optional[CandidatePage8Table] = None
def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks:
"""The rows rows of a per-request publication."""
ids = candidates.ids[rows]
page8 = candidates.page8
if page8 is not None:
page8 = CandidatePage8Table(
page8.ids[rows],
page8.block_tables[rows],
page8.compact_lens[rows],
page8.query_start_loc[: ids.shape[0] + 1],
)
return candidates._replace(
ids=candidates.ids[rows],
ids=ids,
compact_lens=candidates.compact_lens[rows],
compact_page_table=candidates.compact_page_table[rows],
page8=page8,
)
@@ -289,6 +315,7 @@ def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks:
ids=torch.cat([p.ids for p in pieces]),
compact_lens=torch.cat([p.compact_lens for p in pieces]),
compact_page_table=torch.cat([p.compact_page_table for p in pieces]),
page8=None,
)
@@ -389,18 +416,219 @@ def topk_within_candidate_blocks_hip(
"""Level two for a consumer layer: the top-k of logits inside the published candidate
blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with
sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context."""
rows, width = logits.shape
seq_lens = seq_lens.to(torch.int32).contiguous()
compact = gather_candidate_blocks(
logits, seq_lens, candidates.ids, block_size=candidates.block_size
)
topk_compact_candidate_rows(
compact,
candidates.compact_lens,
candidates.ids,
candidates,
seq_lens,
page_table=page_table,
page_size=page_size,
page_indices=page_indices,
raw_indices=raw_indices,
sort_output=sort_output,
)
@triton.jit
def _candidate_page8_table_kernel(
ids_ptr,
seq_lens_ptr,
page_table_ptr,
out_ids_ptr,
pages_ptr,
lens_ptr,
query_start_loc_ptr,
ids_stride,
pt_stride,
out_stride,
n_pages,
K,
BLOCK_SIZE: tl.constexpr,
BLOCKS_PER_PAGE: tl.constexpr,
BLOCK: tl.constexpr,
):
"""One program per row (kept ids a -1 padded prefix): moves the newest, partial block
last so the reachable columns are a prefix, and writes the physical page table and that
exact length."""
row = tl.program_id(0)
j = tl.arange(0, BLOCK)
in_k = j < K
base = ids_ptr + row * ids_stride
ids = tl.load(base + j, mask=in_k, other=-1)
live = ids >= 0
kept = tl.sum(live.to(tl.int32), axis=0)
length = tl.load(seq_lens_ptr + row)
newest = tl.maximum(length - 1, 0) // BLOCK_SIZE
at = tl.max(tl.where(live & (ids == newest), j, -1), axis=0)
last = kept - 1
swap = at >= 0
at_last = tl.sum(tl.where(j == last, ids, 0), axis=0)
ids = tl.where(swap & (j == last), newest, tl.where(swap & (j == at), at_last, ids))
span = tl.where(
swap,
tl.minimum(tl.maximum(length - newest * BLOCK_SIZE, 0), BLOCK_SIZE),
BLOCK_SIZE,
)
tl.store(lens_ptr + row, tl.where(kept > 0, last * BLOCK_SIZE + span, 0))
blk = tl.where(live, ids, 0)
page = tl.load(
page_table_ptr
+ row * pt_stride
+ tl.minimum(blk // BLOCKS_PER_PAGE, n_pages - 1),
mask=live,
other=0,
)
pages = page * BLOCKS_PER_PAGE + blk % BLOCKS_PER_PAGE
tl.store(out_ids_ptr + row * out_stride + j, ids, mask=in_k)
tl.store(pages_ptr + row * out_stride + j, tl.where(live, pages, 0), mask=in_k)
tl.store(query_start_loc_ptr + row + 1, row + 1)
if row == 0:
tl.store(query_start_loc_ptr, 0)
def candidate_page8_table(
ids: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
*,
block_size: int,
page_size: int,
) -> CandidatePage8Table:
"""The CandidatePage8Table of each row's kept block ids."""
rows, k = ids.shape
assert block_size == 8 and page_size % block_size == 0, (block_size, page_size)
assert ids.dtype == torch.int32 and ids.stride(1) == 1
assert page_table.dtype == torch.int32 and page_table.stride(1) == 1
out_ids = torch.empty((rows, k), dtype=torch.int32, device=ids.device)
pages = torch.empty_like(out_ids)
lens = torch.empty(rows, dtype=torch.int32, device=ids.device)
query_start_loc = torch.empty(rows + 1, dtype=torch.int32, device=ids.device)
if not rows:
query_start_loc.zero_()
else:
_candidate_page8_table_kernel[(rows,)](
ids,
seq_lens.to(torch.int32).contiguous(),
page_table,
out_ids,
pages,
lens,
query_start_loc,
ids.stride(0),
page_table.stride(0),
out_ids.stride(0),
page_table.shape[1],
k,
BLOCK_SIZE=block_size,
BLOCKS_PER_PAGE=page_size // block_size,
BLOCK=triton.next_power_of_2(k),
)
return CandidatePage8Table(out_ids, pages, lens, query_start_loc)
def with_candidate_page8_table(
candidates: CandidateBlocks,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
*,
page_size: int,
) -> CandidateBlocks:
"""candidates with their page8 table attached."""
return candidates._replace(
page8=candidate_page8_table(
candidates.ids,
seq_lens,
page_table,
block_size=candidates.block_size,
page_size=page_size,
)
)
def topk_within_candidate_blocks_page8(
*,
q_fp4: torch.Tensor,
q_scale: torch.Tensor,
weights: torch.Tensor,
k_payload: torch.Tensor,
k_scale: torch.Tensor,
candidates: CandidateBlocks,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
page_size: int,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
is_decode: bool = True,
) -> None:
"""topk_within_candidate_blocks_hip on a page8 index-K pool, scoring only the candidate
blocks."""
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
rowgroup_paged_mqa_logits,
)
block_size = candidates.block_size
table = candidates.page8 or candidate_page8_table(
candidates.ids, seq_lens, page_table, block_size=block_size, page_size=page_size
)
compact = rowgroup_paged_mqa_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=k_payload,
k_scale=k_scale,
weights=weights,
weight_scale=1.0,
block_tables=table.block_tables,
query_start_loc=table.query_start_loc,
row_ends=table.compact_lens,
max_query_len=1,
max_seq_len=candidates.ids.shape[1] * block_size,
page8=True,
pages_per_block=1,
is_decode=is_decode,
)
topk_compact_candidate_rows(
compact,
table.compact_lens,
table.ids,
candidates,
seq_lens,
page_table=page_table,
page_size=page_size,
page_indices=page_indices,
raw_indices=raw_indices,
)
def topk_compact_candidate_rows(
compact: torch.Tensor,
compact_lens: torch.Tensor,
ids: torch.Tensor,
candidates: CandidateBlocks,
seq_lens: torch.Tensor,
*,
page_table: torch.Tensor,
page_size: int,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
sort_output: bool = False,
) -> None:
"""The top-k of each compact candidate row (column j * block_size + t is position
ids[j] * block_size + t, valid up to compact_lens), mapped back to pool slots as
topk_within_candidate_blocks_hip writes them."""
rows = compact.shape[0]
topk = page_indices.shape[1]
block_size = candidates.block_size
seq_lens = seq_lens.to(torch.int32).contiguous()
compact = gather_candidate_blocks(
logits, seq_lens, candidates.ids, block_size=block_size
)
assert compact.shape[1] <= candidates.compact_page_size
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=compact.device)
topk_transform_paged_hip(
compact,
candidates.compact_lens,
compact_lens,
candidates.compact_page_table,
compact_pos,
candidates.compact_page_size,
@@ -415,12 +643,12 @@ def topk_within_candidate_blocks_hip(
assert not sort_output or topk & (topk - 1) == 0, topk
_map_compact_selection_kernel[(rows,)](
compact_pos,
candidates.ids,
ids,
seq_lens,
page_table,
page_indices,
raw_indices if write_raw else page_indices,
candidates.ids.stride(0),
ids.stride(0),
page_table.stride(0),
page_indices.stride(0),
page_table.shape[1],
@@ -2,8 +2,10 @@
from __future__ import annotations
import functools
import itertools
import os
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
import torch
import triton
@@ -46,6 +48,15 @@ _PREFILL_BASE_CTA_TARGET = 1024
LOW_RATIO_PAGE_TABLE_BUCKET = 64
# AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len].
_DECODE_CTA_INFO_WIDTH = 4
# Row-group decode: a request's verify rows share each key load from this many requests up;
# below it each row scoring as its own sequence is faster (gfx950, H = 32, 8K..64K context).
_ROWGROUP_MIN_GROUPED_SEQS = 4
# DeepSeek-V4 C4 indexer (H = 64, 64-slot pages): a target-verify scores on the row-group
# kernel, a request's rows one sequence, from this many rows up (4-row DSpark: 16 requests,
# 7-row: 10); below it the prefill kernel's schedule is as fast or faster. MI355X, in a
# DSpark decode step: 7 rows x 16 requests at 131K 1.36x, 4 x 32 per DP rank at 32K 1.11x,
# 4 x 8 at 64K 0.88x (bench_dsv4_c4_indexer_rowgroup_hip.py for the kernel alone).
C4_ROWGROUP_MIN_VERIFY_ROWS = 64
# Budget for the pooled prefill logits block, in MiB. Rows are split to fit it
# (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint
@@ -68,6 +79,17 @@ class FP4DecodeWorkspace(NamedTuple):
schedule_scratch: torch.Tensor
class FP4RowgroupDecodeWorkspace(NamedTuple):
"""Decode workspace of the schedule-free row-group kernel."""
guarded_page_table: torch.Tensor
max_seq_len: int
# sequence b's rows are query_start_loc[b] .. [b + 1] - 1, rows_per_seq each,
# all on page-table row query_start_loc[b]
query_start_loc: torch.Tensor
rows_per_seq: int
class FP4PrefillWorkspace(NamedTuple):
guarded_page_table: torch.Tensor
row_to_batch: torch.Tensor
@@ -81,6 +103,16 @@ class FP4PrefillWorkspace(NamedTuple):
schedule_buffers: Optional[PrefillScheduleBuffers] = None
class FP4RowgroupPrefillWorkspace(NamedTuple):
"""Prefill workspace of the row-group kernel, no schedule: each request's padded
page-table row, and per scored row range the sequences' query_start_loc, built on first
use and shared by the step's layers."""
request_page_table: torch.Tensor
max_seq_len: int
query_start_locs: Dict[Tuple[int, int], torch.Tensor]
class FP4KWriteMetadata(NamedTuple):
positions: torch.Tensor
slots: torch.Tensor
@@ -155,17 +187,32 @@ def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int:
)
def rowgroup_table_columns(
max_context_len: int, ratio: int, page_table_bucket: int = 4
) -> int:
"""Block-table columns of every row-group launch at ``ratio``: one width for the
server's life. AITER compiles the row-group kernel per block-table width and stride,
and its bounded compile cache unloads an evicted variant that a captured CUDA graph
still launches; widths that track the context would also compile on the request path."""
compressed_len = -(-max_context_len // ratio)
pages = -(-compressed_len // _KV_BLOCK_SIZE)
return _guarded_pages(pages, page_table_bucket) + 4
def _guard_page_table(
page_table: torch.Tensor,
out: Optional[torch.Tensor] = None,
page_table_bucket: int = 4,
columns: Optional[int] = None,
):
"""Pad page tables for 256-token scheduling and one-chunk lookahead."""
from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import (
pad_page_table,
)
return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket)
return pad_page_table(
page_table, out=out, page_table_bucket=page_table_bucket, columns=columns
)
def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int:
@@ -244,6 +291,89 @@ def prepare_fp4_decode_workspace(
)
@functools.lru_cache(maxsize=1)
def aiter_has_fp4_rowgroup() -> bool:
"""Whether the installed AITER ships the row-group FP4 MQA logits kernel (#6145)."""
try:
from aiter.ops.flydsl import make_fp4_mqa_plan # noqa: F401
except Exception:
return False
return True
def prepare_fp4_rowgroup_decode_workspace(
page_table: torch.Tensor,
rows_per_request: int,
page_table_bucket: int = 4,
table_columns: Optional[int] = None,
min_grouped_seqs: int = _ROWGROUP_MIN_GROUPED_SEQS,
) -> FP4RowgroupDecodeWorkspace:
"""The decode workspace of the row-group logits kernel: the padded page table and the
rows' sequences, no schedule. ``rows_per_request`` consecutive rows belong to one
request (one page-table row, non-decreasing bounds); they score as one sequence once
there are ``min_grouped_seqs`` requests, else every row is its own. The table has at
least ``table_columns`` columns (``rowgroup_table_columns``). Capture-safe."""
guarded, max_seq_len = _guard_page_table(
page_table, page_table_bucket=page_table_bucket, columns=table_columns
)
rows = guarded.shape[0]
grouped = (
rows_per_request > 1
and rows % rows_per_request == 0
and rows // rows_per_request >= min_grouped_seqs
)
rows_per_seq = rows_per_request if grouped else 1
query_start_loc = torch.arange(
0, rows + 1, rows_per_seq, dtype=torch.int32, device=guarded.device
)
return FP4RowgroupDecodeWorkspace(
guarded, max_seq_len, query_start_loc, rows_per_seq
)
def _pinned_int32(values: List[int], device: torch.device) -> torch.Tensor:
return (
torch.tensor(values, dtype=torch.int32)
.pin_memory()
.to(device, non_blocking=True)
)
def prepare_fp4_rowgroup_prefill_workspace(
page_table: torch.Tensor,
extend_lens: List[int],
page_table_bucket: int = 4,
table_columns: Optional[int] = None,
) -> FP4RowgroupPrefillWorkspace:
"""The row-group prefill workspace of a step whose requests own extend_lens consecutive
page-table rows each, the request table at least ``table_columns`` columns
(``rowgroup_table_columns``)."""
num_rows = page_table.shape[0]
starts = itertools.accumulate([0, *extend_lens[:-1]])
first_rows = [min(s, num_rows - 1) for s in starts]
request_page_table, max_seq_len = _guard_page_table(
page_table.index_select(0, _pinned_int32(first_rows, page_table.device)),
page_table_bucket=page_table_bucket,
columns=table_columns,
)
return FP4RowgroupPrefillWorkspace(request_page_table, max_seq_len, {})
def rowgroup_prefill_query_start_loc(
workspace: FP4RowgroupPrefillWorkspace, rows: slice, rows_per_request: List[int]
) -> torch.Tensor:
"""query_start_loc of the consecutive requests rows_per_request scored as rows."""
key = (rows.start, rows.stop)
query_start_loc = workspace.query_start_locs.get(key)
if query_start_loc is None:
query_start_loc = _pinned_int32(
[0, *itertools.accumulate(rows_per_request)],
workspace.request_page_table.device,
)
workspace.query_start_locs[key] = query_start_loc
return query_start_loc
def prepare_fp4_prefill_workspace(
page_table: torch.Tensor,
c4_seq_lens: torch.Tensor,
@@ -309,6 +439,103 @@ def prepare_fp4_prefill_workspace(
return workspace
def _rowgroup_decode_logits(
*,
q_fp4: torch.Tensor,
q_scale: torch.Tensor,
k_payload: torch.Tensor,
k_scale: torch.Tensor,
weights: torch.Tensor,
weight_scale: float,
workspace: FP4RowgroupDecodeWorkspace,
row_ends: torch.Tensor,
page8: bool = False,
is_decode: bool = True,
) -> torch.Tensor:
"""Decode logits from AITER's row-group kernel over a row-group workspace."""
q = workspace.rows_per_seq
return rowgroup_paged_mqa_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=k_payload,
k_scale=k_scale,
weights=weights,
weight_scale=weight_scale,
block_tables=workspace.guarded_page_table[::q],
query_start_loc=workspace.query_start_loc,
row_ends=row_ends,
max_query_len=q,
max_seq_len=workspace.max_seq_len,
page8=page8,
pages_per_block=8 if page8 else 1,
is_decode=is_decode,
)
def rowgroup_paged_mqa_logits(
*,
q_fp4: torch.Tensor,
q_scale: torch.Tensor,
k_payload: torch.Tensor,
k_scale: torch.Tensor,
weights: torch.Tensor,
weight_scale: float,
block_tables: torch.Tensor,
query_start_loc: torch.Tensor,
row_ends: torch.Tensor,
max_query_len: int,
max_seq_len: int,
page8: bool,
pages_per_block: int = 1,
is_decode: bool = True,
) -> torch.Tensor:
"""[rows, max_seq_len] fp32 logits from AITER's row-group kernel: sequence b's rows
query_start_loc[b] .. [b + 1] - 1 score the keys of block_tables[b] up to their own
row_ends. An entry names a 64-slot page of the pool, or with page8 pages_per_block of
the pool's 8-slot pages (8: the 64-slot page table as is; 1: an 8-slot page each, the
candidate table). Past a row's bound the logits are left unwritten; the length-aware
top-k never reads them."""
from aiter.ops.flydsl import flydsl_pa_mqa_logits_fp4, make_fp4_mqa_plan
num_rows, heads = q_fp4.shape[0], q_fp4.shape[-2]
if page8:
kv, ks = index_k_page8_views(k_payload, k_scale)
page_size = 8
else:
assert pages_per_block == 1, "a 64-slot page entry names one page"
kv, ks = k_payload.view(torch.uint8), k_scale
page_size = _KV_BLOCK_SIZE
plan = make_fp4_mqa_plan(
num_seqs=query_start_loc.shape[0] - 1,
max_qlen=max_query_len,
num_rows=num_rows,
heads=heads,
page_size=page_size,
max_seq_len=max_seq_len,
pages_per_block=pages_per_block,
)
logits = _alloc_logits(num_rows, max_seq_len, q_fp4.device, is_decode)
flydsl_pa_mqa_logits_fp4(
q_fp4.view(torch.uint8).reshape(num_rows, 1, heads, _HEAD_DIM // 2),
q_scale.reshape(num_rows, 1, *_Q_SCALE_SHAPE),
kv,
ks,
block_tables,
weights,
None,
max_seq_len,
weight_scale=weight_scale,
kv_block_size=page_size,
out=logits,
row_ends=_as_int32_1d(row_ends),
query_start_loc=query_start_loc,
max_query_len=max_query_len,
pages_per_block=pages_per_block,
plan=plan,
)
return logits
def aiter_fp4_paged_mqa_logits(
*,
q_fp4: torch.Tensor,
@@ -320,11 +547,17 @@ def aiter_fp4_paged_mqa_logits(
c4_seq_lens: torch.Tensor,
weight_scale: float,
is_decode: bool,
decode_workspace: Optional[FP4DecodeWorkspace] = None,
decode_workspace: Optional[
Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
] = None,
prefill_workspace: Optional[FP4PrefillWorkspace] = None,
page_table_bucket: int = 4,
page8: bool = False,
rowgroup_table_columns: Optional[int] = None,
) -> torch.Tensor:
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel."""
"""Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel. A page8 pool
is read by the row-group kernel only: without a row-group decode workspace every row
scores as its own sequence, on a table of ``rowgroup_table_columns`` columns."""
from aiter.ops.flydsl import (
flydsl_pa_mqa_logits_fp4,
flydsl_pa_mqa_logits_fp4_prefill,
@@ -336,7 +569,42 @@ def aiter_fp4_paged_mqa_logits(
# A workspace is bound to one row count. DP padding or truncated activations
# can leave it stale, in which case fall back to building the schedule here.
if workspace is not None and workspace.guarded_page_table.shape[0] != num_tokens:
workspace = None
if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace):
# Stay on the row-group kernel, capture-safe and on the same compiled table
# width, scoring every row as its own sequence.
workspace = prepare_fp4_rowgroup_decode_workspace(
page_table,
1,
page_table_bucket=page_table_bucket,
table_columns=workspace.guarded_page_table.shape[1],
)
else:
workspace = None
rowgroup_ws = (
workspace
if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace)
else None
)
if page8 and rowgroup_ws is None:
rowgroup_ws = prepare_fp4_rowgroup_decode_workspace(
page_table,
1,
page_table_bucket=page_table_bucket,
table_columns=rowgroup_table_columns,
)
if rowgroup_ws is not None:
return _rowgroup_decode_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=k_payload,
k_scale=k_scale,
weights=weights,
weight_scale=weight_scale,
workspace=rowgroup_ws,
row_ends=c4_seq_lens,
page8=page8,
is_decode=is_decode,
)
# Built on the fallback path below; kept in scope so the schedule scratch
# outlives the logits kernel that reads it.
fallback_schedule = None
@@ -555,6 +823,7 @@ def _store_fp4_index_k_split_kernel(
loc,
page_size: tl.constexpr,
BLOCK: tl.constexpr,
PAGE8: tl.constexpr,
):
token_id = tl.program_id(0)
cache_loc = tl.load(loc + token_id)
@@ -566,6 +835,23 @@ def _store_fp4_index_k_split_kernel(
chunk = offsets // 16
byte = offsets - chunk * 16
k = tl.load(k_fp4 + token_id * BLOCK + offsets)
sf = tl.load(k_sf + token_id)
sf_offsets = tl.arange(0, 4)
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
if PAGE8:
# aiter's row-group 8-slot page: payload [t % 4][chunk][t // 4][16 B], scales [chunk][t]
page8 = cache_loc // 8
t = cache_loc % 8
tl.store(
payload
+ page8 * (8 * BLOCK)
+ ((t % 4) * 4 + chunk) * 32
+ (t // 4) * 16
+ byte,
k,
)
tl.store(scale + page8 * 32 + sf_offsets * 8 + t, sf_bytes)
return
tl.store(
payload
+ page * (4 * page_size * 16)
@@ -576,9 +862,6 @@ def _store_fp4_index_k_split_kernel(
)
# scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI)
shuffled = (page_offset % 16) * 4 + page_offset // 16
sf = tl.load(k_sf + token_id)
sf_offsets = tl.arange(0, 4)
sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8)
tl.store(
scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes
)
@@ -592,10 +875,12 @@ def store_fp4_index_k_cache_split(
*,
page_size: int,
rne: bool = False,
page8: bool = False,
) -> None:
"""Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot
loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16],
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed)."""
scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed); with page8
each page holds eight 8-slot pages of aiter's row-group layout (pack_kv_cache)."""
assert input.shape[-1] == _HEAD_DIM
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
@@ -612,17 +897,43 @@ def store_fp4_index_k_cache_split(
loc,
page_size,
BLOCK=64,
PAGE8=page8,
)
def index_k_page8_views(
payload: torch.Tensor, scale: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""A page8 pool's buffers as aiter's 8-slot pages: (payload uint8 [pages * 8, 512],
scale uint8 [pages * 8, 32]); 8-slot page p covers slots 8p .. 8p + 7."""
pages8 = payload.shape[0] * (payload.shape[3] // 8)
return (
payload.view(torch.uint8).view(pages8, 8 * _HEAD_DIM // 2),
scale.view(pages8, 8 * _HEAD_DIM // 32),
)
def read_fp4_index_k_split(
payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int
payload: torch.Tensor,
scale: torch.Tensor,
slots: torch.Tensor,
*,
page_size: int,
page8: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales
int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of
quantize_fp4_indexer_tensor."""
assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page"
slots = slots.to(torch.int64)
if page8:
kv8, ks8 = index_k_page8_views(payload, scale)
page, t = slots // 8, slots % 8
kv8 = kv8.view(-1, 4, 4, 2, 16) # [page8, t % 4, chunk, t // 4, 16]
rows = kv8[page, t % 4, :, t // 4, :].reshape(-1, 64).view(torch.int8)
sf = ks8.view(-1, 4, 8)[page, :, t].to(torch.int32) # [n, 4]
packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24)
return rows, packed
page, off = slots // page_size, slots % page_size
rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16]
rows = rows.reshape(-1, 64).view(torch.int8)
@@ -21,9 +21,11 @@ if TYPE_CHECKING:
@cache_once
def _jit_index_k_split_module(
head_dim: int, rope_dim: int, page_size: int, ratio: int
head_dim: int, rope_dim: int, page_size: int, ratio: int, page8: bool
) -> Module:
args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl())
args = make_cpp_args(
head_dim, rope_dim, page_size, ratio, page8, is_arch_support_pdl()
)
return load_jit(
make_name("index_k_rope_pack_split"),
*args,
@@ -43,16 +45,18 @@ def index_k_norm_rope_pack_store_split(
scale: torch.Tensor,
*,
ratio: int,
page8: bool = False,
) -> None:
"""`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split
FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4,
page_size] uint8); the bytes equal store_fp4_index_k_cache_split's."""
page_size] uint8), with page8 each page as eight row-group 8-slot pages; the bytes
equal store_fp4_index_k_cache_split's."""
head_dim = input.shape[-1]
page_size = payload.shape[3]
assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape
assert scale.shape[1:] == (1, 4, page_size), scale.shape
_jit_index_k_split_module(
head_dim, freqs_cis.shape[-1], page_size, ratio
head_dim, freqs_cis.shape[-1], page_size, ratio, page8
).index_k_split(
input,
norm_weight,
@@ -198,21 +198,24 @@ def pad_page_table(
page_table: torch.Tensor,
out: Optional[torch.Tensor] = None,
page_table_bucket: int = 4,
columns: Optional[int] = None,
) -> Tuple[torch.Tensor, int]:
"""Pad a page table for 256-token scheduling in a single dispatch.
Replaces the ``new_zeros`` + masked ``copy_`` pair: the kernel writes every
output element, so the destination never needs pre-zeroing.
output element, so the destination never needs pre-zeroing. ``columns`` widens
the destination to at least that many columns; the returned logits width stays
the table's own.
"""
page_table = _as_int32_2d(page_table)
rows, logical_width, padded_width = padded_page_table_shape(
page_table, page_table_bucket
)
w_dst = max(padded_width + 4, columns or 0)
if out is None:
out = page_table.new_empty((rows, padded_width + 4))
out = page_table.new_empty((rows, w_dst))
else:
assert out.shape == (rows, padded_width + 4), f"{out.shape=} {rows=}"
w_dst = padded_width + 4
assert out.shape == (rows, w_dst), f"{out.shape=} {rows=}"
if rows:
_pad_page_table_kernel[(rows, triton.cdiv(w_dst, _PT_BLOCK))](
page_table,
+10
View File
@@ -1688,6 +1688,16 @@ class Envs:
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
# V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group
# FP4 MQA logits kernel, a request's verify rows sharing each key load.
SGLANG_HIP_FP4_INDEXER_ROWGROUP = EnvBool(_default_hip)
# DeepSeek-V4 C4 decode / target-verify indexer: the same row-group kernel (aiter >= #6145),
# its workspace built in the graph without a schedule; a request's verify rows share each
# key load.
SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP = EnvBool(False)
# V4.1 ratio-1 index-K pool in aiter's row-group page-8 layout (8 consecutive 8-slot pages per
# 64-slot page), so a candidate consumer layer scores only its published blocks.
SGLANG_HIP_DSV41_INDEX_K_PAGE8 = EnvBool(False)
# cache, GEMM, and distributed
SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True)
@@ -65,6 +65,7 @@ from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
LateLayerTail,
_as_int_list,
_prefill_graph_max_seq_len,
_tail_rows,
)
@@ -120,6 +121,8 @@ if TYPE_CHECKING:
FP4DecodeWorkspace,
FP4KWriteMetadata,
FP4PrefillWorkspace,
FP4RowgroupDecodeWorkspace,
FP4RowgroupPrefillWorkspace,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -684,15 +687,17 @@ class DSV4Metadata:
# Per-step scratch for the TP-padded query heads (models/deepseek_v4.py).
q_pad_buffer: Optional[torch.Tensor] = None
# low-ratio FlyDSL indexer workspaces by ratio; address-pinned like the c4 ones, rebuilt in place
fp4_low_ratio_decode_workspaces: Dict[int, FP4DecodeWorkspace] = field(
default_factory=dict, repr=False
)
fp4_low_ratio_prefill_workspaces: Dict[int, FP4PrefillWorkspace] = field(
default_factory=dict, repr=False
)
fp4_low_ratio_decode_workspaces: Dict[
int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
] = field(default_factory=dict, repr=False)
fp4_low_ratio_prefill_workspaces: Dict[
int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace]
] = field(default_factory=dict, repr=False)
# Captured kernels bind these FP4 indexer buffers by address; absent from copy_
# so addresses stay pinned across replays (the builders refresh contents).
fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False)
fp4_decode_workspace: Optional[
Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
] = field(default=None, repr=False)
fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field(
default=None, repr=False
)
@@ -1015,6 +1020,36 @@ class DeepseekV4HipRadixBackend(
)
self._fp4_graph_row_limit = MAX_FUSED_ROWS
# DSv4 C4 decode / target-verify indexer on AITER's row-group kernel: no schedule,
# its workspace built in the graph, a request's verify rows sharing each key load.
# Every launch reads a table of one width (its compiled variant) for the server's life.
self._fp4_c4_rowgroup_columns: Optional[int] = None
if (
self.enable_deepseek_v4_fp4_indexer
and self.speculative_num_steps == 0
and self.has_c4
and envs.SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP.get()
):
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
aiter_has_fp4_rowgroup,
rowgroup_table_columns,
)
if aiter_has_fp4_rowgroup():
self._fp4_c4_rowgroup_columns = rowgroup_table_columns(
max(self.max_context_len, self.MAX_SEQ_LEN_FOR_CAPTURE), 4
)
logger.info(
"DSv4 C4 indexer: decode / target-verify on the row-group FP4 "
"MQA logits kernel (%d table columns)",
self._fp4_c4_rowgroup_columns,
)
else:
logger.warning(
"SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP is set but the installed AITER "
"has no row-group FP4 MQA logits kernel (#6145); keeping the "
"schedule-based C4 indexer."
)
self.speculative_step_id = speculative_step_id
self.forward_metadata: Union[
DSV4Metadata,
@@ -1728,13 +1763,29 @@ class DeepseekV4HipRadixBackend(
if not low_ratio_decode_rows_are_identity(
self, forward_batch, ratio
)
}
},
max_context_len=self.max_context_len,
num_requests=forward_batch.batch_size,
page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(),
)
)
# Decode's schedule builder is capture-safe (the workspace pins its scratch);
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace.
if self._fp4_workspaces_enabled(metadata) and (
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. The
# row-group workspace has no schedule, so target-verify builds it here too.
rowgroup_rows = self._fp4_c4_rowgroup_rows(metadata, forward_batch)
if rowgroup_rows is not None:
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
prepare_fp4_rowgroup_decode_workspace,
)
metadata.fp4_decode_workspace = prepare_fp4_rowgroup_decode_workspace(
metadata.indexer_metadata.page_table,
rowgroup_rows,
table_columns=self._fp4_c4_rowgroup_columns,
min_grouped_seqs=1,
)
elif self._fp4_workspaces_enabled(metadata) and (
forward_batch.forward_mode.is_decode()
):
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
@@ -1758,6 +1809,40 @@ class DeepseekV4HipRadixBackend(
and metadata.c4_compress_metadata is not None
)
def _fp4_c4_rowgroup_rows(
self, metadata, forward_batch: ForwardBatch
) -> Optional[int]:
"""Rows a request scores as one sequence on the row-group kernel this step: 1 for
decode, the draft rows for a target-verify of at least C4_ROWGROUP_MIN_VERIFY_ROWS
rows (below it the prefill kernel's schedule spreads a few requests' keys wider).
None keeps the schedule kernels, as does a verify whose logits outgrow the prefill
budget (the row-group launch is unchunked).
"""
if self._fp4_c4_rowgroup_columns is None or not self._fp4_workspaces_enabled(
metadata
):
return None
mode = forward_batch.forward_mode
if mode.is_decode():
return 1
if not mode.is_target_verify():
return None
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
C4_ROWGROUP_MIN_VERIFY_ROWS,
logits_rows_per_chunk,
)
page_table = metadata.indexer_metadata.page_table
rows, num_draft = page_table.shape[0], self.target_verify_num_draft_tokens
if (
num_draft < 2
or rows != num_draft * forward_batch.batch_size
or rows < C4_ROWGROUP_MIN_VERIFY_ROWS
or rows > logits_rows_per_chunk(page_table)
):
return None
return num_draft
def _refresh_fp4_prefill_workspace(
self, forward_batch: ForwardBatch, metadata: Optional[DSV4Metadata] = None
) -> None:
@@ -1775,6 +1860,13 @@ class DeepseekV4HipRadixBackend(
refresh_low_ratio_prefill_workspaces(
metadata.low_ratio_indexer_metadata_by_ratio(),
metadata.fp4_low_ratio_prefill_workspaces,
extend_lens=(
metadata.late_layer_tail.extend_seq_lens_cpu
if metadata.late_layer_tail is not None
else _as_int_list(forward_batch.extend_seq_lens_cpu)
),
max_context_len=self.max_context_len,
page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(),
)
)
if not self._fp4_workspaces_enabled(metadata):
@@ -1785,6 +1877,9 @@ class DeepseekV4HipRadixBackend(
ForwardMode.TARGET_VERIFY,
):
return
if self._fp4_c4_rowgroup_rows(metadata, forward_batch) is not None:
# scored on the row-group kernel, its workspace built with the metadata
return
if (
get_parallel().attn_cp_size != 1
or getattr(forward_batch, "tbo_children", None)
@@ -24,6 +24,7 @@ from sglang.kernels.ops.attention.dsv4 import (
topk_transform_paged_v2,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
FP4RowgroupDecodeWorkspace,
aiter_fp4_paged_mqa_logits,
aiter_q_indexer_fp4,
logits_rows_per_chunk,
@@ -1018,6 +1019,14 @@ class C4IndexerBackendMixin:
elif use_aiter_fp4:
q_fp4, q_scale = q
is_decode = forward_batch.forward_mode.is_decode()
if isinstance(
getattr(metadata, "fp4_decode_workspace", None),
FP4RowgroupDecodeWorkspace,
) and (is_decode or forward_batch.forward_mode.is_target_verify()):
# The backend built a row-group workspace for this step: target-verify
# rows score like decode rows, one launch over the whole batch, a
# request's rows sharing each key load.
is_decode = True
# Hoisted: these await this layer's KV transfer, which every chunk
# would otherwise re-await.
k_payload = token_to_kv_pool.get_index_k_fp4_payload_buffer(
@@ -3,8 +3,19 @@ payload / scale index-K pools, then the top-k v2 transform -- the DeepGEMM path'
from __future__ import annotations
import functools
import itertools
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
import logging
from typing import (
TYPE_CHECKING,
Callable,
Dict,
FrozenSet,
List,
Optional,
Tuple,
Union,
)
import torch
import triton
@@ -17,12 +28,17 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
slice_candidate_blocks,
topk_transform_paged_hip,
topk_within_candidate_blocks_hip,
topk_within_candidate_blocks_page8,
with_candidate_page8_table,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
LOW_RATIO_PAGE_TABLE_BUCKET,
FP4DecodeWorkspace,
FP4PrefillWorkspace,
FP4RowgroupDecodeWorkspace,
FP4RowgroupPrefillWorkspace,
aiter_fp4_paged_mqa_logits,
aiter_has_fp4_rowgroup,
index_q_rope_pack_flydsl,
index_q_rope_pack_weights_flydsl,
indexer_head_weights,
@@ -30,11 +46,17 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
pack_fp4_query_flydsl,
prepare_fp4_decode_workspace,
prepare_fp4_prefill_workspace,
prepare_fp4_rowgroup_decode_workspace,
prepare_fp4_rowgroup_prefill_workspace,
rowgroup_paged_mqa_logits,
rowgroup_prefill_query_start_loc,
rowgroup_table_columns,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import (
index_k_norm_rope_pack_store_split,
)
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
from sglang.srt.environ import envs
from sglang.srt.layers.attention.deepseek_v4_backend import (
_as_int_list,
)
@@ -47,6 +69,61 @@ if TYPE_CHECKING:
# fp32 logits scored per candidate-block chunk stay under this many bytes
_CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30
logger = logging.getLogger(__name__)
@functools.lru_cache(maxsize=1)
def _fp4_rowgroup_enabled() -> bool:
"""SGLANG_HIP_FP4_INDEXER_ROWGROUP, off when the installed AITER has no row-group
kernel (ROCm/aiter#6145)."""
if not envs.SGLANG_HIP_FP4_INDEXER_ROWGROUP.get():
return False
if not aiter_has_fp4_rowgroup():
logger.warning(
"SGLANG_HIP_FP4_INDEXER_ROWGROUP is set but the installed AITER has no "
"row-group FP4 MQA logits kernel (#6145); keeping the schedule kernels."
)
return False
return True
def _score_requests_page8(
*,
q_fp4: torch.Tensor,
q_scale: torch.Tensor,
weights: torch.Tensor,
k_payload: torch.Tensor,
k_scale: torch.Tensor,
workspace: FP4RowgroupPrefillWorkspace,
rows: slice,
req_lo: int,
rows_per_request: List[int],
row_ends: torch.Tensor,
) -> torch.Tensor:
"""Full-context logits of a page8 pool for rows, consecutive requests from req_lo on with
rows_per_request rows each; a request's rows share every key load."""
assert sum(rows_per_request) == q_fp4.shape[0], (rows_per_request, q_fp4.shape)
return rowgroup_paged_mqa_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=k_payload,
k_scale=k_scale,
weights=weights,
weight_scale=1.0,
block_tables=workspace.request_page_table[
req_lo : req_lo + len(rows_per_request)
],
query_start_loc=rowgroup_prefill_query_start_loc(
workspace, rows, rows_per_request
),
row_ends=row_ends,
max_query_len=max(rows_per_request),
max_seq_len=workspace.max_seq_len,
page8=True,
pages_per_block=8,
is_decode=False,
)
def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool:
"""Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch."""
@@ -114,33 +191,74 @@ def _indexer_inputs(layer, x, q_lora, pos):
def build_low_ratio_decode_workspaces(
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
) -> Dict[int, FP4DecodeWorkspace]:
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace."""
return {
ratio: prepare_fp4_decode_workspace(
meta.page_table,
meta.compressed_seq_lens,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
*,
max_context_len: int,
num_requests: Optional[int] = None,
page8_ratios: FrozenSet[int] = frozenset(),
) -> Dict[int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]]:
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace.
With the row-group kernel there is no schedule; a request's rows (``num_requests``
requests of equal row counts, consecutive: decode 1, target-verify its draft rows)
score as one sequence, on a table as wide as ``max_context_len`` needs. A page8 pool
is read by the row-group kernel only."""
rowgroup = _fp4_rowgroup_enabled()
workspaces = {}
for ratio, meta in metadata_by_ratio.items():
if not (rowgroup or ratio in page8_ratios):
workspaces[ratio] = prepare_fp4_decode_workspace(
meta.page_table,
meta.compressed_seq_lens,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
)
continue
rows = meta.page_table.shape[0]
rows_per_request = (
rows // num_requests if num_requests and rows % num_requests == 0 else 1
)
for ratio, meta in metadata_by_ratio.items()
}
workspaces[ratio] = prepare_fp4_rowgroup_decode_workspace(
meta.page_table,
rows_per_request,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
table_columns=rowgroup_table_columns(
max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
),
)
return workspaces
def refresh_low_ratio_prefill_workspaces(
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
previous: Optional[Dict[int, FP4PrefillWorkspace]],
) -> Dict[int, FP4PrefillWorkspace]:
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace."""
*,
extend_lens: Optional[List[int]],
max_context_len: int,
page8_ratios: FrozenSet[int] = frozenset(),
) -> Dict[int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace]]:
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace. A page8 pool
scores on the row-group kernel, which takes no schedule, on a table as wide as
``max_context_len`` needs (without extend_lens, the indexer builds its workspace
itself)."""
previous = previous or {}
return {
ratio: prepare_fp4_prefill_workspace(
workspaces = {}
for ratio, meta in metadata_by_ratio.items():
if ratio in page8_ratios:
if extend_lens is not None:
workspaces[ratio] = prepare_fp4_rowgroup_prefill_workspace(
meta.page_table,
extend_lens,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
table_columns=rowgroup_table_columns(
max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
),
)
continue
workspaces[ratio] = prepare_fp4_prefill_workspace(
meta.page_table,
meta.compressed_seq_lens,
workspace=previous.get(ratio),
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
)
for ratio, meta in metadata_by_ratio.items()
}
return workspaces
def low_ratio_identity_skip_enabled(
@@ -277,11 +395,37 @@ def low_ratio_index_topk_hip_decode(
two_level = False
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
page8 = pool.low_ratio_index_k_page8(layer.layer_id)
if two_level and indexer.uses_candidates:
candidates = backend.candidate_masks
assert (
isinstance(candidates, CandidateBlocks)
and candidates.ids.shape[0] == q_fp4.shape[0]
and candidates.block_size == indexer.candidate_block_size
), "candidate blocks missing for decode"
if page8:
topk_within_candidate_blocks_page8(
q_fp4=q_fp4,
q_scale=q_scale,
weights=weights,
k_payload=k_payload,
k_scale=k_scale,
candidates=candidates,
seq_lens=indexer_metadata.compressed_seq_lens,
page_table=indexer_metadata.page_table,
page_size=indexer_metadata.compressed_page_size,
page_indices=core.sparse_page_indices(ratio),
raw_indices=core.sparse_raw_indices(ratio),
is_decode=True,
)
return
logits = aiter_fp4_paged_mqa_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id),
k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id),
k_payload=k_payload,
k_scale=k_scale,
weights=weights,
page_table=indexer_metadata.page_table,
c4_seq_lens=indexer_metadata.compressed_seq_lens,
@@ -289,15 +433,13 @@ def low_ratio_index_topk_hip_decode(
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
is_decode=True,
decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio),
page8=page8,
rowgroup_table_columns=rowgroup_table_columns(
backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
),
)
# level one is bounded on device by the compressed lengths: a captured step cannot read them back
if two_level and indexer.uses_candidates:
candidates = backend.candidate_masks
assert (
isinstance(candidates, CandidateBlocks)
and candidates.ids.shape[0] == logits.shape[0]
and candidates.block_size == indexer.candidate_block_size
), "candidate blocks missing for decode"
topk_within_candidate_blocks_hip(
logits,
indexer_metadata.compressed_seq_lens,
@@ -309,12 +451,20 @@ def low_ratio_index_topk_hip_decode(
)
return
if two_level and indexer.is_candidate_source:
backend.candidate_masks = select_candidate_blocks_hip(
candidates = select_candidate_blocks_hip(
logits,
indexer_metadata.compressed_seq_lens,
topk_blocks=indexer.candidate_topk_blocks,
block_size=indexer.candidate_block_size,
)
if page8:
candidates = with_candidate_page8_table(
candidates,
indexer_metadata.compressed_seq_lens,
indexer_metadata.page_table,
page_size=indexer_metadata.compressed_page_size,
)
backend.candidate_masks = candidates
topk_transform_paged_hip(
logits,
indexer_metadata.compressed_seq_lens,
@@ -426,8 +576,36 @@ def low_ratio_index_topk_hip_extend(
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio)
page8 = pool.low_ratio_index_k_page8(layer.layer_id)
if page8 and not (
isinstance(prefill_workspace, FP4RowgroupPrefillWorkspace)
and prefill_workspace.request_page_table.shape[0] == len(extend_lens_cpu)
):
prefill_workspace = prepare_fp4_rowgroup_prefill_workspace(
indexer_metadata.page_table,
extend_lens_cpu,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
table_columns=rowgroup_table_columns(
backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET
),
)
def score_rows(rows: slice) -> torch.Tensor:
def score_rows(
rows: slice, req_lo: int, rows_per_request: List[int]
) -> torch.Tensor:
if page8:
return _score_requests_page8(
q_fp4=q_fp4[rows],
q_scale=q_scale[rows],
weights=weights[rows],
k_payload=k_payload,
k_scale=k_scale,
workspace=prefill_workspace,
rows=rows,
req_lo=req_lo,
rows_per_request=rows_per_request,
row_ends=indexer_metadata.compressed_seq_lens[rows],
)
return aiter_fp4_paged_mqa_logits(
q_fp4=q_fp4[rows],
q_scale=q_scale[rows],
@@ -442,16 +620,58 @@ def low_ratio_index_topk_hip_extend(
prefill_workspace=prefill_workspace if rows.start == 0 else None,
)
def select_rows_page8_consumer(rows: slice, req_lo, rows_per_request, consume_rows):
"""A consumer on a page8 pool scores only a request's candidate blocks."""
def full_logits(r: slice, b: int) -> torch.Tensor:
g = slice(rows.start + r.start, rows.start + r.stop)
return score_rows(g, req_lo + b, [r.stop - r.start])
def within(r: slice, candidates, rows_page, rows_raw) -> None:
g = slice(rows.start + r.start, rows.start + r.stop)
topk_within_candidate_blocks_page8(
q_fp4=q_fp4[g],
q_scale=q_scale[g],
weights=weights[g],
k_payload=k_payload,
k_scale=k_scale,
candidates=candidates,
seq_lens=compress_lens[g],
page_table=indexer_metadata.page_table[g],
page_size=indexer_metadata.compressed_page_size,
page_indices=rows_page,
raw_indices=rows_raw,
is_decode=False,
)
_consume_candidates_per_request(
extend_lens_cpu=rows_per_request,
consume=consume_rows,
compress_lens=compress_lens[rows],
page_table=indexer_metadata.page_table[rows],
page_size=indexer_metadata.compressed_page_size,
page_indices=page_indices[rows],
raw_indices=raw_indices[rows] if raw_indices is not None else None,
full_logits=full_logits,
within=within,
)
def select_rows(
rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish
):
rows_per_request = (
[rows.stop - rows.start]
if req_hi == req_lo + 1
else extend_lens_cpu[req_lo:req_hi]
)
if page8 and consume_rows is not None:
select_rows_page8_consumer(rows, req_lo, rows_per_request, consume_rows)
return
_select_topk_extend_hip(
indexer=indexer,
logits=score_rows(rows),
logits=score_rows(rows, req_lo, rows_per_request),
lc_per_req=lc_per_req[req_lo:req_hi],
extend_lens_cpu=[rows.stop - rows.start]
if req_hi == req_lo + 1
else extend_lens_cpu[req_lo:req_hi],
extend_lens_cpu=rows_per_request,
is_identity=group_is_identity,
compress_lens=compress_lens[rows],
page_table=indexer_metadata.page_table[rows],
@@ -503,6 +723,19 @@ def low_ratio_index_topk_hip_extend(
if publish is not None:
publish.append(cat_candidate_blocks(pieces))
if publish is not None:
if page8:
assert len(publish) == len(extend_lens_cpu), (len(publish), extend_lens_cpu)
tok = 0
for b, candidates in enumerate(publish):
r = slice(tok, tok + extend_lens_cpu[b])
tok = r.stop
if candidates is not None:
publish[b] = with_candidate_page8_table(
candidates,
compress_lens[r],
indexer_metadata.page_table[r],
page_size=indexer_metadata.compressed_page_size,
)
backend.candidate_masks = publish
@@ -551,33 +784,27 @@ def _select_topk_extend_hip(
)
publish.append(cat_candidate_blocks(pieces))
if consume is not None:
tok_start = 0
for b, t_len in enumerate(extend_lens_cpu):
rows = slice(tok_start, tok_start + t_len)
tok_start += t_len
if t_len == 0:
continue
rows_page = page_indices[rows]
rows_raw = raw_indices[rows] if raw_indices is not None else None
if consume[b] is None:
topk_transform_paged_hip(
logits[rows],
compress_lens[rows].contiguous(),
page_table[rows],
rows_page,
page_size,
rows_raw,
)
else:
_consume_candidates_per_request(
extend_lens_cpu=extend_lens_cpu,
consume=consume,
compress_lens=compress_lens,
page_table=page_table,
page_size=page_size,
page_indices=page_indices,
raw_indices=raw_indices,
full_logits=lambda rows, b: logits[rows],
within=lambda rows, candidates, rows_page, rows_raw: (
topk_within_candidate_blocks_hip(
logits[rows],
compress_lens[rows],
consume[b],
candidates,
page_table=page_table[rows],
page_size=page_size,
page_indices=rows_page,
raw_indices=rows_raw,
)
),
)
return
topk_transform_paged_hip(
logits,
@@ -589,6 +816,43 @@ def _select_topk_extend_hip(
)
def _consume_candidates_per_request(
*,
extend_lens_cpu: List[int],
consume: List[Optional[CandidateBlocks]],
compress_lens: torch.Tensor,
page_table: torch.Tensor,
page_size: int,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
full_logits: Callable[[slice, int], torch.Tensor],
within: Callable[
[slice, CandidateBlocks, torch.Tensor, Optional[torch.Tensor]], None
],
) -> None:
"""A consumer layer's rows, request by request: within(rows, candidates, ...) selects inside
the published blocks; a request with none runs the paged top-k of full_logits(rows, b)."""
tok_start = 0
for b, t_len in enumerate(extend_lens_cpu):
rows = slice(tok_start, tok_start + t_len)
tok_start += t_len
if t_len == 0:
continue
rows_page = page_indices[rows]
rows_raw = raw_indices[rows] if raw_indices is not None else None
if consume[b] is None:
topk_transform_paged_hip(
full_logits(rows, b),
compress_lens[rows].contiguous(),
page_table[rows],
rows_page,
page_size,
rows_raw,
)
else:
within(rows, consume[b], rows_page, rows_raw)
def _request_groups(
extend_lens_cpu: List[int], rows_per_chunk: int
) -> List[Tuple[int, int, int, int]]:
@@ -620,4 +884,5 @@ def store_index_k_split(
pool.get_index_k_fp4_payload_buffer(layer_id),
pool.get_index_k_fp4_scale_buffer(layer_id),
ratio=layer.compress_ratio,
page8=pool.low_ratio_index_k_page8(layer_id),
)
@@ -2,7 +2,16 @@ from __future__ import annotations
import logging
from contextlib import nullcontext
from typing import List, Literal, NamedTuple, Optional, Sequence, Tuple, Union
from typing import (
FrozenSet,
List,
Literal,
NamedTuple,
Optional,
Sequence,
Tuple,
Union,
)
import torch
@@ -557,6 +566,10 @@ class DeepSeekV4IndexerPool(KVCache):
self.uses_aiter_fp4_layout = _is_hip and self.use_fp4_indexer
# Low-ratio pools round to nearest even; c4 keeps threshold rounding.
self.index_k_rne = False
# aiter's row-group page-8 layout inside each page (pack_kv_cache): page size and shapes
# are unchanged but the bytes are not, so PD peers must use the same layout and a
# persisted cache written under the other one must be cleared before switching
self.index_k_page8 = False
self._create_buffer()
@@ -702,6 +715,7 @@ class DeepSeekV4IndexerPool(KVCache):
loc,
page_size=self.page_size,
rne=self.index_k_rne,
page8=self.index_k_page8,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
store_fp4_index_k_cache,
@@ -1533,6 +1547,19 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
kv_layout=self.compressed_kv_layout(ratio),
)
index_k_page8 = _is_hip and envs.SGLANG_HIP_DSV41_INDEX_K_PAGE8.get()
if index_k_page8:
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
aiter_has_fp4_rowgroup,
)
if not aiter_has_fp4_rowgroup():
logger.warning(
"SGLANG_HIP_DSV41_INDEX_K_PAGE8 is set but the installed AITER has "
"no row-group FP4 MQA logits kernel (#6145); keeping the page-64 "
"index-K pool."
)
index_k_page8 = False
self.index_pools: dict[int, DeepSeekV4IndexerPool] = {
ratio: self._make_indexer_pool(
config.indexer_size,
@@ -1559,6 +1586,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
device,
enable_memory_saver,
force_fp4=True,
# only the candidate consumers (ratio 1) score a subset of blocks
index_k_page8=ratio == 1 and index_k_page8,
)
# HiCache and hardware backends still read these per-ratio attributes.
@@ -1616,6 +1645,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
enable_memory_saver: bool,
force_fp4: bool = False,
global_page_size: Optional[int] = None,
index_k_page8: bool = False,
) -> DeepSeekV4IndexerPool:
"""Build the c4 lightning-indexer K pool (packed CUDA layout).
Overridden by :class:`DSV4NPUTokenToKVPool` to swap in the
@@ -1636,6 +1666,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
)
# The dsv41 low-ratio indexer rounds to nearest even (reference rounding).
pool.index_k_rne = True
pool.index_k_page8 = index_k_page8
return pool
return DeepSeekV4IndexerPool(
size,
@@ -1998,6 +2029,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
compress_ratio, _, _ = self.layer_mapping[layer_id]
return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout
def low_ratio_index_k_page8(self, layer_id: int) -> bool:
"""Whether the layer's split index-K pool lays each page out as row-group 8-slot pages."""
compress_ratio, _, _ = self.layer_mapping[layer_id]
return self._indexer_pool(compress_ratio).index_k_page8
def low_ratio_index_k_page8_ratios(self) -> FrozenSet[int]:
"""The compression ratios whose index-K pool is in the page-8 layout."""
return frozenset(
ratio for ratio, pool in self.index_pools.items() if pool.index_k_page8
)
def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor:
self.wait_layer_transfer(layer_id)
compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id]