diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh index a702e09130..f157d5a1cc 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/fp4_indexer_rope_hip.cuh @@ -1,6 +1,8 @@ /// Index-K write of `fp4_indexer_rope.cuh` into the split FlyDSL layout: payload `[npages, 1, 4, kPageSize, 16]` /// (chunk `c` holds elements `[32c, 32c + 32)`) and ue8m0 exponents `[npages, 1, 4, kPageSize]` with /// the slot axis transposed as a 16 x 4 tile -- the bytes `store_fp4_index_k_cache_split` writes. +/// With kPage8 the same bytes of a page hold eight 8-slot pages in aiter's row-group layout +/// (`pack_kv_cache(page_size=8)`): payload `[8, 4 (t % 4), 4 (chunk), 2 (t / 4), 16]`, scales `[8, 4, 8]`. #pragma once @@ -24,7 +26,14 @@ struct IndexKSplitParams { float eps; }; -template +template < + bool kUsePDL, + int64_t kHeadDim, + int64_t kRopeDim, + uint32_t kPageSize, + uint32_t kRatio, + bool kPage8, + typename PosT> __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void flash_index_k_split_kernel( const IndexKSplitParams params) { using namespace device; @@ -75,6 +84,23 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla // slot 0 is reserved: padded graph rows and, at ratio > 1, rows completing no group publish nothing if (slot_id <= 0) return; + if constexpr (kPage8) { + constexpr int64_t kP8 = 8; + const auto page8 = slot_id / kP8; + const auto t = slot_id % kP8; + // payload [t % 4][chunk][t / 4][16 B] and scales [chunk][t] of the slot's 8-slot page + const auto payload_ptr = params.payload + page8 * (kP8 * kHeadDim / 2) + (t / 4) * 16; + const auto chunk_row = (t % 4) * 4; + payload_ptr[(chunk_row + lane / 16) * 32 + lane % 16] = static_cast(packed.payload[0]); + payload_ptr[(chunk_row + 2 + lane / 16) * 32 + lane % 16] = static_cast(packed.payload[1]); + if (lane % (kWarpThreads / 2) == 0) { + const auto scale_ptr = params.scale + page8 * (kP8 * kHeadDim / 32) + t; + const auto first = lane / (kWarpThreads / 2); + scale_ptr[first * kP8] = static_cast(packed.exponent[0]); + scale_ptr[(2 + first) * kP8] = static_cast(packed.exponent[1]); + } + return; + } const auto page = slot_id / kPageSize; const auto slot = slot_id % kPageSize; @@ -94,12 +120,13 @@ __global__ __launch_bounds__(kFp4RopeWarpsPerCTA* device::kWarpThreads) void fla } /// \brief Host side of `flash_index_k_split_kernel`. -template +template struct FlashIndexKSplitKernel { static constexpr uint32_t kBlockSize = kFp4RopeWarpsPerCTA * device::kWarpThreads; template - static constexpr auto kernel = flash_index_k_split_kernel; + static constexpr auto kernel = + flash_index_k_split_kernel; /// `IndexKKernel::run_index_k`'s arguments, the cache replaced by /// \param payload `[npages, 1, 4, kPageSize, 16]` uint8 (the pool's diff --git a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py index 766c818e34..8f16f38466 100644 --- a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py @@ -259,6 +259,20 @@ def _map_compact_selection_kernel( tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid) +class CandidatePage8Table(NamedTuple): + """The candidate blocks of each row as a page-8 pool's row-group sequence: one row per + sequence, its blocks one 8-slot page each, the valid columns a prefix.""" + + # int32 [rows, topk_blocks]: the kept ids, the newest (partial) block last, -1 padded + ids: torch.Tensor + # int32 [rows, topk_blocks]: the 8-slot pool page of each id, 0 padded + block_tables: torch.Tensor + # int32 [rows]: the reachable columns of the compact row + compact_lens: torch.Tensor + # int32 [rows + 1]: arange, one row a sequence + query_start_loc: torch.Tensor + + class CandidateBlocks(NamedTuple): """What the candidate-source layer publishes for the decode rows of a step.""" @@ -270,14 +284,26 @@ class CandidateBlocks(NamedTuple): compact_page_table: torch.Tensor compact_page_size: int block_size: int + # on a page8 pool, built once and shared by the consumer layers + page8: Optional[CandidatePage8Table] = None def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks: """The rows rows of a per-request publication.""" + ids = candidates.ids[rows] + page8 = candidates.page8 + if page8 is not None: + page8 = CandidatePage8Table( + page8.ids[rows], + page8.block_tables[rows], + page8.compact_lens[rows], + page8.query_start_loc[: ids.shape[0] + 1], + ) return candidates._replace( - ids=candidates.ids[rows], + ids=ids, compact_lens=candidates.compact_lens[rows], compact_page_table=candidates.compact_page_table[rows], + page8=page8, ) @@ -289,6 +315,7 @@ def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks: ids=torch.cat([p.ids for p in pieces]), compact_lens=torch.cat([p.compact_lens for p in pieces]), compact_page_table=torch.cat([p.compact_page_table for p in pieces]), + page8=None, ) @@ -389,18 +416,219 @@ def topk_within_candidate_blocks_hip( """Level two for a consumer layer: the top-k of logits inside the published candidate blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context.""" - rows, width = logits.shape + seq_lens = seq_lens.to(torch.int32).contiguous() + compact = gather_candidate_blocks( + logits, seq_lens, candidates.ids, block_size=candidates.block_size + ) + topk_compact_candidate_rows( + compact, + candidates.compact_lens, + candidates.ids, + candidates, + seq_lens, + page_table=page_table, + page_size=page_size, + page_indices=page_indices, + raw_indices=raw_indices, + sort_output=sort_output, + ) + + +@triton.jit +def _candidate_page8_table_kernel( + ids_ptr, + seq_lens_ptr, + page_table_ptr, + out_ids_ptr, + pages_ptr, + lens_ptr, + query_start_loc_ptr, + ids_stride, + pt_stride, + out_stride, + n_pages, + K, + BLOCK_SIZE: tl.constexpr, + BLOCKS_PER_PAGE: tl.constexpr, + BLOCK: tl.constexpr, +): + """One program per row (kept ids a -1 padded prefix): moves the newest, partial block + last so the reachable columns are a prefix, and writes the physical page table and that + exact length.""" + row = tl.program_id(0) + j = tl.arange(0, BLOCK) + in_k = j < K + base = ids_ptr + row * ids_stride + ids = tl.load(base + j, mask=in_k, other=-1) + live = ids >= 0 + kept = tl.sum(live.to(tl.int32), axis=0) + length = tl.load(seq_lens_ptr + row) + newest = tl.maximum(length - 1, 0) // BLOCK_SIZE + at = tl.max(tl.where(live & (ids == newest), j, -1), axis=0) + last = kept - 1 + swap = at >= 0 + at_last = tl.sum(tl.where(j == last, ids, 0), axis=0) + ids = tl.where(swap & (j == last), newest, tl.where(swap & (j == at), at_last, ids)) + span = tl.where( + swap, + tl.minimum(tl.maximum(length - newest * BLOCK_SIZE, 0), BLOCK_SIZE), + BLOCK_SIZE, + ) + tl.store(lens_ptr + row, tl.where(kept > 0, last * BLOCK_SIZE + span, 0)) + blk = tl.where(live, ids, 0) + page = tl.load( + page_table_ptr + + row * pt_stride + + tl.minimum(blk // BLOCKS_PER_PAGE, n_pages - 1), + mask=live, + other=0, + ) + pages = page * BLOCKS_PER_PAGE + blk % BLOCKS_PER_PAGE + tl.store(out_ids_ptr + row * out_stride + j, ids, mask=in_k) + tl.store(pages_ptr + row * out_stride + j, tl.where(live, pages, 0), mask=in_k) + tl.store(query_start_loc_ptr + row + 1, row + 1) + if row == 0: + tl.store(query_start_loc_ptr, 0) + + +def candidate_page8_table( + ids: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + *, + block_size: int, + page_size: int, +) -> CandidatePage8Table: + """The CandidatePage8Table of each row's kept block ids.""" + rows, k = ids.shape + assert block_size == 8 and page_size % block_size == 0, (block_size, page_size) + assert ids.dtype == torch.int32 and ids.stride(1) == 1 + assert page_table.dtype == torch.int32 and page_table.stride(1) == 1 + out_ids = torch.empty((rows, k), dtype=torch.int32, device=ids.device) + pages = torch.empty_like(out_ids) + lens = torch.empty(rows, dtype=torch.int32, device=ids.device) + query_start_loc = torch.empty(rows + 1, dtype=torch.int32, device=ids.device) + if not rows: + query_start_loc.zero_() + else: + _candidate_page8_table_kernel[(rows,)]( + ids, + seq_lens.to(torch.int32).contiguous(), + page_table, + out_ids, + pages, + lens, + query_start_loc, + ids.stride(0), + page_table.stride(0), + out_ids.stride(0), + page_table.shape[1], + k, + BLOCK_SIZE=block_size, + BLOCKS_PER_PAGE=page_size // block_size, + BLOCK=triton.next_power_of_2(k), + ) + return CandidatePage8Table(out_ids, pages, lens, query_start_loc) + + +def with_candidate_page8_table( + candidates: CandidateBlocks, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + *, + page_size: int, +) -> CandidateBlocks: + """candidates with their page8 table attached.""" + return candidates._replace( + page8=candidate_page8_table( + candidates.ids, + seq_lens, + page_table, + block_size=candidates.block_size, + page_size=page_size, + ) + ) + + +def topk_within_candidate_blocks_page8( + *, + q_fp4: torch.Tensor, + q_scale: torch.Tensor, + weights: torch.Tensor, + k_payload: torch.Tensor, + k_scale: torch.Tensor, + candidates: CandidateBlocks, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + page_size: int, + page_indices: torch.Tensor, + raw_indices: Optional[torch.Tensor], + is_decode: bool = True, +) -> None: + """topk_within_candidate_blocks_hip on a page8 index-K pool, scoring only the candidate + blocks.""" + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + rowgroup_paged_mqa_logits, + ) + + block_size = candidates.block_size + table = candidates.page8 or candidate_page8_table( + candidates.ids, seq_lens, page_table, block_size=block_size, page_size=page_size + ) + compact = rowgroup_paged_mqa_logits( + q_fp4=q_fp4, + q_scale=q_scale, + k_payload=k_payload, + k_scale=k_scale, + weights=weights, + weight_scale=1.0, + block_tables=table.block_tables, + query_start_loc=table.query_start_loc, + row_ends=table.compact_lens, + max_query_len=1, + max_seq_len=candidates.ids.shape[1] * block_size, + page8=True, + pages_per_block=1, + is_decode=is_decode, + ) + topk_compact_candidate_rows( + compact, + table.compact_lens, + table.ids, + candidates, + seq_lens, + page_table=page_table, + page_size=page_size, + page_indices=page_indices, + raw_indices=raw_indices, + ) + + +def topk_compact_candidate_rows( + compact: torch.Tensor, + compact_lens: torch.Tensor, + ids: torch.Tensor, + candidates: CandidateBlocks, + seq_lens: torch.Tensor, + *, + page_table: torch.Tensor, + page_size: int, + page_indices: torch.Tensor, + raw_indices: Optional[torch.Tensor], + sort_output: bool = False, +) -> None: + """The top-k of each compact candidate row (column j * block_size + t is position + ids[j] * block_size + t, valid up to compact_lens), mapped back to pool slots as + topk_within_candidate_blocks_hip writes them.""" + rows = compact.shape[0] topk = page_indices.shape[1] block_size = candidates.block_size seq_lens = seq_lens.to(torch.int32).contiguous() - compact = gather_candidate_blocks( - logits, seq_lens, candidates.ids, block_size=block_size - ) assert compact.shape[1] <= candidates.compact_page_size - compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device) + compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=compact.device) topk_transform_paged_hip( compact, - candidates.compact_lens, + compact_lens, candidates.compact_page_table, compact_pos, candidates.compact_page_size, @@ -415,12 +643,12 @@ def topk_within_candidate_blocks_hip( assert not sort_output or topk & (topk - 1) == 0, topk _map_compact_selection_kernel[(rows,)]( compact_pos, - candidates.ids, + ids, seq_lens, page_table, page_indices, raw_indices if write_raw else page_indices, - candidates.ids.stride(0), + ids.stride(0), page_table.stride(0), page_indices.stride(0), page_table.shape[1], diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py index 9a56eec586..8b957c65a5 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py @@ -2,8 +2,10 @@ from __future__ import annotations +import functools +import itertools import os -from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union import torch import triton @@ -46,6 +48,15 @@ _PREFILL_BASE_CTA_TARGET = 1024 LOW_RATIO_PAGE_TABLE_BUCKET = 64 # AITER varctx cta_info row: [batch_packed, chunk_start, chunk_count, ctx_len]. _DECODE_CTA_INFO_WIDTH = 4 +# Row-group decode: a request's verify rows share each key load from this many requests up; +# below it each row scoring as its own sequence is faster (gfx950, H = 32, 8K..64K context). +_ROWGROUP_MIN_GROUPED_SEQS = 4 +# DeepSeek-V4 C4 indexer (H = 64, 64-slot pages): a target-verify scores on the row-group +# kernel, a request's rows one sequence, from this many rows up (4-row DSpark: 16 requests, +# 7-row: 10); below it the prefill kernel's schedule is as fast or faster. MI355X, in a +# DSpark decode step: 7 rows x 16 requests at 131K 1.36x, 4 x 32 per DP rank at 32K 1.11x, +# 4 x 8 at 64K 0.88x (bench_dsv4_c4_indexer_rowgroup_hip.py for the kernel alone). +C4_ROWGROUP_MIN_VERIFY_ROWS = 64 # Budget for the pooled prefill logits block, in MiB. Rows are split to fit it # (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint @@ -68,6 +79,17 @@ class FP4DecodeWorkspace(NamedTuple): schedule_scratch: torch.Tensor +class FP4RowgroupDecodeWorkspace(NamedTuple): + """Decode workspace of the schedule-free row-group kernel.""" + + guarded_page_table: torch.Tensor + max_seq_len: int + # sequence b's rows are query_start_loc[b] .. [b + 1] - 1, rows_per_seq each, + # all on page-table row query_start_loc[b] + query_start_loc: torch.Tensor + rows_per_seq: int + + class FP4PrefillWorkspace(NamedTuple): guarded_page_table: torch.Tensor row_to_batch: torch.Tensor @@ -81,6 +103,16 @@ class FP4PrefillWorkspace(NamedTuple): schedule_buffers: Optional[PrefillScheduleBuffers] = None +class FP4RowgroupPrefillWorkspace(NamedTuple): + """Prefill workspace of the row-group kernel, no schedule: each request's padded + page-table row, and per scored row range the sequences' query_start_loc, built on first + use and shared by the step's layers.""" + + request_page_table: torch.Tensor + max_seq_len: int + query_start_locs: Dict[Tuple[int, int], torch.Tensor] + + class FP4KWriteMetadata(NamedTuple): positions: torch.Tensor slots: torch.Tensor @@ -155,17 +187,32 @@ def _guarded_pages(logical_width: int, page_table_bucket: int = 4) -> int: ) +def rowgroup_table_columns( + max_context_len: int, ratio: int, page_table_bucket: int = 4 +) -> int: + """Block-table columns of every row-group launch at ``ratio``: one width for the + server's life. AITER compiles the row-group kernel per block-table width and stride, + and its bounded compile cache unloads an evicted variant that a captured CUDA graph + still launches; widths that track the context would also compile on the request path.""" + compressed_len = -(-max_context_len // ratio) + pages = -(-compressed_len // _KV_BLOCK_SIZE) + return _guarded_pages(pages, page_table_bucket) + 4 + + def _guard_page_table( page_table: torch.Tensor, out: Optional[torch.Tensor] = None, page_table_bucket: int = 4, + columns: Optional[int] = None, ): """Pad page tables for 256-token scheduling and one-chunk lookahead.""" from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( pad_page_table, ) - return pad_page_table(page_table, out=out, page_table_bucket=page_table_bucket) + return pad_page_table( + page_table, out=out, page_table_bucket=page_table_bucket, columns=columns + ) def logits_rows_per_chunk(page_table: torch.Tensor, page_table_bucket: int = 4) -> int: @@ -244,6 +291,89 @@ def prepare_fp4_decode_workspace( ) +@functools.lru_cache(maxsize=1) +def aiter_has_fp4_rowgroup() -> bool: + """Whether the installed AITER ships the row-group FP4 MQA logits kernel (#6145).""" + try: + from aiter.ops.flydsl import make_fp4_mqa_plan # noqa: F401 + except Exception: + return False + return True + + +def prepare_fp4_rowgroup_decode_workspace( + page_table: torch.Tensor, + rows_per_request: int, + page_table_bucket: int = 4, + table_columns: Optional[int] = None, + min_grouped_seqs: int = _ROWGROUP_MIN_GROUPED_SEQS, +) -> FP4RowgroupDecodeWorkspace: + """The decode workspace of the row-group logits kernel: the padded page table and the + rows' sequences, no schedule. ``rows_per_request`` consecutive rows belong to one + request (one page-table row, non-decreasing bounds); they score as one sequence once + there are ``min_grouped_seqs`` requests, else every row is its own. The table has at + least ``table_columns`` columns (``rowgroup_table_columns``). Capture-safe.""" + guarded, max_seq_len = _guard_page_table( + page_table, page_table_bucket=page_table_bucket, columns=table_columns + ) + rows = guarded.shape[0] + grouped = ( + rows_per_request > 1 + and rows % rows_per_request == 0 + and rows // rows_per_request >= min_grouped_seqs + ) + rows_per_seq = rows_per_request if grouped else 1 + query_start_loc = torch.arange( + 0, rows + 1, rows_per_seq, dtype=torch.int32, device=guarded.device + ) + return FP4RowgroupDecodeWorkspace( + guarded, max_seq_len, query_start_loc, rows_per_seq + ) + + +def _pinned_int32(values: List[int], device: torch.device) -> torch.Tensor: + return ( + torch.tensor(values, dtype=torch.int32) + .pin_memory() + .to(device, non_blocking=True) + ) + + +def prepare_fp4_rowgroup_prefill_workspace( + page_table: torch.Tensor, + extend_lens: List[int], + page_table_bucket: int = 4, + table_columns: Optional[int] = None, +) -> FP4RowgroupPrefillWorkspace: + """The row-group prefill workspace of a step whose requests own extend_lens consecutive + page-table rows each, the request table at least ``table_columns`` columns + (``rowgroup_table_columns``).""" + num_rows = page_table.shape[0] + starts = itertools.accumulate([0, *extend_lens[:-1]]) + first_rows = [min(s, num_rows - 1) for s in starts] + request_page_table, max_seq_len = _guard_page_table( + page_table.index_select(0, _pinned_int32(first_rows, page_table.device)), + page_table_bucket=page_table_bucket, + columns=table_columns, + ) + return FP4RowgroupPrefillWorkspace(request_page_table, max_seq_len, {}) + + +def rowgroup_prefill_query_start_loc( + workspace: FP4RowgroupPrefillWorkspace, rows: slice, rows_per_request: List[int] +) -> torch.Tensor: + """query_start_loc of the consecutive requests rows_per_request scored as rows.""" + key = (rows.start, rows.stop) + query_start_loc = workspace.query_start_locs.get(key) + if query_start_loc is None: + query_start_loc = _pinned_int32( + [0, *itertools.accumulate(rows_per_request)], + workspace.request_page_table.device, + ) + workspace.query_start_locs[key] = query_start_loc + return query_start_loc + + def prepare_fp4_prefill_workspace( page_table: torch.Tensor, c4_seq_lens: torch.Tensor, @@ -309,6 +439,103 @@ def prepare_fp4_prefill_workspace( return workspace +def _rowgroup_decode_logits( + *, + q_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_payload: torch.Tensor, + k_scale: torch.Tensor, + weights: torch.Tensor, + weight_scale: float, + workspace: FP4RowgroupDecodeWorkspace, + row_ends: torch.Tensor, + page8: bool = False, + is_decode: bool = True, +) -> torch.Tensor: + """Decode logits from AITER's row-group kernel over a row-group workspace.""" + q = workspace.rows_per_seq + return rowgroup_paged_mqa_logits( + q_fp4=q_fp4, + q_scale=q_scale, + k_payload=k_payload, + k_scale=k_scale, + weights=weights, + weight_scale=weight_scale, + block_tables=workspace.guarded_page_table[::q], + query_start_loc=workspace.query_start_loc, + row_ends=row_ends, + max_query_len=q, + max_seq_len=workspace.max_seq_len, + page8=page8, + pages_per_block=8 if page8 else 1, + is_decode=is_decode, + ) + + +def rowgroup_paged_mqa_logits( + *, + q_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_payload: torch.Tensor, + k_scale: torch.Tensor, + weights: torch.Tensor, + weight_scale: float, + block_tables: torch.Tensor, + query_start_loc: torch.Tensor, + row_ends: torch.Tensor, + max_query_len: int, + max_seq_len: int, + page8: bool, + pages_per_block: int = 1, + is_decode: bool = True, +) -> torch.Tensor: + """[rows, max_seq_len] fp32 logits from AITER's row-group kernel: sequence b's rows + query_start_loc[b] .. [b + 1] - 1 score the keys of block_tables[b] up to their own + row_ends. An entry names a 64-slot page of the pool, or with page8 pages_per_block of + the pool's 8-slot pages (8: the 64-slot page table as is; 1: an 8-slot page each, the + candidate table). Past a row's bound the logits are left unwritten; the length-aware + top-k never reads them.""" + from aiter.ops.flydsl import flydsl_pa_mqa_logits_fp4, make_fp4_mqa_plan + + num_rows, heads = q_fp4.shape[0], q_fp4.shape[-2] + if page8: + kv, ks = index_k_page8_views(k_payload, k_scale) + page_size = 8 + else: + assert pages_per_block == 1, "a 64-slot page entry names one page" + kv, ks = k_payload.view(torch.uint8), k_scale + page_size = _KV_BLOCK_SIZE + plan = make_fp4_mqa_plan( + num_seqs=query_start_loc.shape[0] - 1, + max_qlen=max_query_len, + num_rows=num_rows, + heads=heads, + page_size=page_size, + max_seq_len=max_seq_len, + pages_per_block=pages_per_block, + ) + logits = _alloc_logits(num_rows, max_seq_len, q_fp4.device, is_decode) + flydsl_pa_mqa_logits_fp4( + q_fp4.view(torch.uint8).reshape(num_rows, 1, heads, _HEAD_DIM // 2), + q_scale.reshape(num_rows, 1, *_Q_SCALE_SHAPE), + kv, + ks, + block_tables, + weights, + None, + max_seq_len, + weight_scale=weight_scale, + kv_block_size=page_size, + out=logits, + row_ends=_as_int32_1d(row_ends), + query_start_loc=query_start_loc, + max_query_len=max_query_len, + pages_per_block=pages_per_block, + plan=plan, + ) + return logits + + def aiter_fp4_paged_mqa_logits( *, q_fp4: torch.Tensor, @@ -320,11 +547,17 @@ def aiter_fp4_paged_mqa_logits( c4_seq_lens: torch.Tensor, weight_scale: float, is_decode: bool, - decode_workspace: Optional[FP4DecodeWorkspace] = None, + decode_workspace: Optional[ + Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace] + ] = None, prefill_workspace: Optional[FP4PrefillWorkspace] = None, page_table_bucket: int = 4, + page8: bool = False, + rowgroup_table_columns: Optional[int] = None, ) -> torch.Tensor: - """Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel.""" + """Compute FP4 Q/K indexer logits with the decode or prefill FlyDSL kernel. A page8 pool + is read by the row-group kernel only: without a row-group decode workspace every row + scores as its own sequence, on a table of ``rowgroup_table_columns`` columns.""" from aiter.ops.flydsl import ( flydsl_pa_mqa_logits_fp4, flydsl_pa_mqa_logits_fp4_prefill, @@ -336,7 +569,42 @@ def aiter_fp4_paged_mqa_logits( # A workspace is bound to one row count. DP padding or truncated activations # can leave it stale, in which case fall back to building the schedule here. if workspace is not None and workspace.guarded_page_table.shape[0] != num_tokens: - workspace = None + if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace): + # Stay on the row-group kernel, capture-safe and on the same compiled table + # width, scoring every row as its own sequence. + workspace = prepare_fp4_rowgroup_decode_workspace( + page_table, + 1, + page_table_bucket=page_table_bucket, + table_columns=workspace.guarded_page_table.shape[1], + ) + else: + workspace = None + rowgroup_ws = ( + workspace + if is_decode and isinstance(workspace, FP4RowgroupDecodeWorkspace) + else None + ) + if page8 and rowgroup_ws is None: + rowgroup_ws = prepare_fp4_rowgroup_decode_workspace( + page_table, + 1, + page_table_bucket=page_table_bucket, + table_columns=rowgroup_table_columns, + ) + if rowgroup_ws is not None: + return _rowgroup_decode_logits( + q_fp4=q_fp4, + q_scale=q_scale, + k_payload=k_payload, + k_scale=k_scale, + weights=weights, + weight_scale=weight_scale, + workspace=rowgroup_ws, + row_ends=c4_seq_lens, + page8=page8, + is_decode=is_decode, + ) # Built on the fallback path below; kept in scope so the schedule scratch # outlives the logits kernel that reads it. fallback_schedule = None @@ -555,6 +823,7 @@ def _store_fp4_index_k_split_kernel( loc, page_size: tl.constexpr, BLOCK: tl.constexpr, + PAGE8: tl.constexpr, ): token_id = tl.program_id(0) cache_loc = tl.load(loc + token_id) @@ -566,6 +835,23 @@ def _store_fp4_index_k_split_kernel( chunk = offsets // 16 byte = offsets - chunk * 16 k = tl.load(k_fp4 + token_id * BLOCK + offsets) + sf = tl.load(k_sf + token_id) + sf_offsets = tl.arange(0, 4) + sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8) + if PAGE8: + # aiter's row-group 8-slot page: payload [t % 4][chunk][t // 4][16 B], scales [chunk][t] + page8 = cache_loc // 8 + t = cache_loc % 8 + tl.store( + payload + + page8 * (8 * BLOCK) + + ((t % 4) * 4 + chunk) * 32 + + (t // 4) * 16 + + byte, + k, + ) + tl.store(scale + page8 * 32 + sf_offsets * 8 + t, sf_bytes) + return tl.store( payload + page * (4 * page_size * 16) @@ -576,9 +862,6 @@ def _store_fp4_index_k_split_kernel( ) # scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI) shuffled = (page_offset % 16) * 4 + page_offset // 16 - sf = tl.load(k_sf + token_id) - sf_offsets = tl.arange(0, 4) - sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8) tl.store( scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes ) @@ -592,10 +875,12 @@ def store_fp4_index_k_cache_split( *, page_size: int, rne: bool = False, + page8: bool = False, ) -> None: """Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16], - scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed).""" + scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed); with page8 + each page holds eight 8-slot pages of aiter's row-group layout (pack_kv_cache).""" assert input.shape[-1] == _HEAD_DIM assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape @@ -612,17 +897,43 @@ def store_fp4_index_k_cache_split( loc, page_size, BLOCK=64, + PAGE8=page8, + ) + + +def index_k_page8_views( + payload: torch.Tensor, scale: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """A page8 pool's buffers as aiter's 8-slot pages: (payload uint8 [pages * 8, 512], + scale uint8 [pages * 8, 32]); 8-slot page p covers slots 8p .. 8p + 7.""" + pages8 = payload.shape[0] * (payload.shape[3] // 8) + return ( + payload.view(torch.uint8).view(pages8, 8 * _HEAD_DIM // 2), + scale.view(pages8, 8 * _HEAD_DIM // 32), ) def read_fp4_index_k_split( - payload: torch.Tensor, scale: torch.Tensor, slots: torch.Tensor, *, page_size: int + payload: torch.Tensor, + scale: torch.Tensor, + slots: torch.Tensor, + *, + page_size: int, + page8: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: """Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of quantize_fp4_indexer_tensor.""" assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" slots = slots.to(torch.int64) + if page8: + kv8, ks8 = index_k_page8_views(payload, scale) + page, t = slots // 8, slots % 8 + kv8 = kv8.view(-1, 4, 4, 2, 16) # [page8, t % 4, chunk, t // 4, 16] + rows = kv8[page, t % 4, :, t // 4, :].reshape(-1, 64).view(torch.int8) + sf = ks8.view(-1, 4, 8)[page, :, t].to(torch.int32) # [n, 4] + packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) + return rows, packed page, off = slots // page_size, slots % page_size rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16] rows = rows.reshape(-1, 64).view(torch.int8) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py index 0d02c5fbc1..43046e5bce 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_rope_hip.py @@ -21,9 +21,11 @@ if TYPE_CHECKING: @cache_once def _jit_index_k_split_module( - head_dim: int, rope_dim: int, page_size: int, ratio: int + head_dim: int, rope_dim: int, page_size: int, ratio: int, page8: bool ) -> Module: - args = make_cpp_args(head_dim, rope_dim, page_size, ratio, is_arch_support_pdl()) + args = make_cpp_args( + head_dim, rope_dim, page_size, ratio, page8, is_arch_support_pdl() + ) return load_jit( make_name("index_k_rope_pack_split"), *args, @@ -43,16 +45,18 @@ def index_k_norm_rope_pack_store_split( scale: torch.Tensor, *, ratio: int, + page8: bool = False, ) -> None: """`fp4_indexer_rope.index_k_norm_rope_pack_store` into the split FlyDSL K layout (payload [npages, 1, 4, page_size, 16], scale [npages, 1, 4, - page_size] uint8); the bytes equal store_fp4_index_k_cache_split's.""" + page_size] uint8), with page8 each page as eight row-group 8-slot pages; the bytes + equal store_fp4_index_k_cache_split's.""" head_dim = input.shape[-1] page_size = payload.shape[3] assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape assert scale.shape[1:] == (1, 4, page_size), scale.shape _jit_index_k_split_module( - head_dim, freqs_cis.shape[-1], page_size, ratio + head_dim, freqs_cis.shape[-1], page_size, ratio, page8 ).index_k_split( input, norm_weight, diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py index f6fe697a2c..5e7025302b 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_schedule_hip.py @@ -198,21 +198,24 @@ def pad_page_table( page_table: torch.Tensor, out: Optional[torch.Tensor] = None, page_table_bucket: int = 4, + columns: Optional[int] = None, ) -> Tuple[torch.Tensor, int]: """Pad a page table for 256-token scheduling in a single dispatch. Replaces the ``new_zeros`` + masked ``copy_`` pair: the kernel writes every - output element, so the destination never needs pre-zeroing. + output element, so the destination never needs pre-zeroing. ``columns`` widens + the destination to at least that many columns; the returned logits width stays + the table's own. """ page_table = _as_int32_2d(page_table) rows, logical_width, padded_width = padded_page_table_shape( page_table, page_table_bucket ) + w_dst = max(padded_width + 4, columns or 0) if out is None: - out = page_table.new_empty((rows, padded_width + 4)) + out = page_table.new_empty((rows, w_dst)) else: - assert out.shape == (rows, padded_width + 4), f"{out.shape=} {rows=}" - w_dst = padded_width + 4 + assert out.shape == (rows, w_dst), f"{out.shape=} {rows=}" if rows: _pad_page_table_kernel[(rows, triton.cdiv(w_dst, _PT_BLOCK))]( page_table, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index ae45e1c3a5..ff8f8ddf8a 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1688,6 +1688,16 @@ class Envs: SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip) SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip) SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip) + # V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group + # FP4 MQA logits kernel, a request's verify rows sharing each key load. + SGLANG_HIP_FP4_INDEXER_ROWGROUP = EnvBool(_default_hip) + # DeepSeek-V4 C4 decode / target-verify indexer: the same row-group kernel (aiter >= #6145), + # its workspace built in the graph without a schedule; a request's verify rows share each + # key load. + SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP = EnvBool(False) + # V4.1 ratio-1 index-K pool in aiter's row-group page-8 layout (8 consecutive 8-slot pages per + # 64-slot page), so a candidate consumer layer scores only its published blocks. + SGLANG_HIP_DSV41_INDEX_K_PAGE8 = EnvBool(False) # cache, GEMM, and distributed SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 98593b84c0..cec3ebf141 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -65,6 +65,7 @@ from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, LateLayerTail, + _as_int_list, _prefill_graph_max_seq_len, _tail_rows, ) @@ -120,6 +121,8 @@ if TYPE_CHECKING: FP4DecodeWorkspace, FP4KWriteMetadata, FP4PrefillWorkspace, + FP4RowgroupDecodeWorkspace, + FP4RowgroupPrefillWorkspace, ) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.model_executor.model_runner import ModelRunner @@ -684,15 +687,17 @@ class DSV4Metadata: # Per-step scratch for the TP-padded query heads (models/deepseek_v4.py). q_pad_buffer: Optional[torch.Tensor] = None # low-ratio FlyDSL indexer workspaces by ratio; address-pinned like the c4 ones, rebuilt in place - fp4_low_ratio_decode_workspaces: Dict[int, FP4DecodeWorkspace] = field( - default_factory=dict, repr=False - ) - fp4_low_ratio_prefill_workspaces: Dict[int, FP4PrefillWorkspace] = field( - default_factory=dict, repr=False - ) + fp4_low_ratio_decode_workspaces: Dict[ + int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace] + ] = field(default_factory=dict, repr=False) + fp4_low_ratio_prefill_workspaces: Dict[ + int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace] + ] = field(default_factory=dict, repr=False) # Captured kernels bind these FP4 indexer buffers by address; absent from copy_ # so addresses stay pinned across replays (the builders refresh contents). - fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False) + fp4_decode_workspace: Optional[ + Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace] + ] = field(default=None, repr=False) fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field( default=None, repr=False ) @@ -1015,6 +1020,36 @@ class DeepseekV4HipRadixBackend( ) self._fp4_graph_row_limit = MAX_FUSED_ROWS + # DSv4 C4 decode / target-verify indexer on AITER's row-group kernel: no schedule, + # its workspace built in the graph, a request's verify rows sharing each key load. + # Every launch reads a table of one width (its compiled variant) for the server's life. + self._fp4_c4_rowgroup_columns: Optional[int] = None + if ( + self.enable_deepseek_v4_fp4_indexer + and self.speculative_num_steps == 0 + and self.has_c4 + and envs.SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP.get() + ): + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + aiter_has_fp4_rowgroup, + rowgroup_table_columns, + ) + + if aiter_has_fp4_rowgroup(): + self._fp4_c4_rowgroup_columns = rowgroup_table_columns( + max(self.max_context_len, self.MAX_SEQ_LEN_FOR_CAPTURE), 4 + ) + logger.info( + "DSv4 C4 indexer: decode / target-verify on the row-group FP4 " + "MQA logits kernel (%d table columns)", + self._fp4_c4_rowgroup_columns, + ) + else: + logger.warning( + "SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP is set but the installed AITER " + "has no row-group FP4 MQA logits kernel (#6145); keeping the " + "schedule-based C4 indexer." + ) self.speculative_step_id = speculative_step_id self.forward_metadata: Union[ DSV4Metadata, @@ -1728,13 +1763,29 @@ class DeepseekV4HipRadixBackend( if not low_ratio_decode_rows_are_identity( self, forward_batch, ratio ) - } + }, + max_context_len=self.max_context_len, + num_requests=forward_batch.batch_size, + page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(), ) ) # Decode's schedule builder is capture-safe (the workspace pins its scratch); - # prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. - if self._fp4_workspaces_enabled(metadata) and ( + # prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. The + # row-group workspace has no schedule, so target-verify builds it here too. + rowgroup_rows = self._fp4_c4_rowgroup_rows(metadata, forward_batch) + if rowgroup_rows is not None: + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + prepare_fp4_rowgroup_decode_workspace, + ) + + metadata.fp4_decode_workspace = prepare_fp4_rowgroup_decode_workspace( + metadata.indexer_metadata.page_table, + rowgroup_rows, + table_columns=self._fp4_c4_rowgroup_columns, + min_grouped_seqs=1, + ) + elif self._fp4_workspaces_enabled(metadata) and ( forward_batch.forward_mode.is_decode() ): from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( @@ -1758,6 +1809,40 @@ class DeepseekV4HipRadixBackend( and metadata.c4_compress_metadata is not None ) + def _fp4_c4_rowgroup_rows( + self, metadata, forward_batch: ForwardBatch + ) -> Optional[int]: + """Rows a request scores as one sequence on the row-group kernel this step: 1 for + decode, the draft rows for a target-verify of at least C4_ROWGROUP_MIN_VERIFY_ROWS + rows (below it the prefill kernel's schedule spreads a few requests' keys wider). + None keeps the schedule kernels, as does a verify whose logits outgrow the prefill + budget (the row-group launch is unchunked). + """ + if self._fp4_c4_rowgroup_columns is None or not self._fp4_workspaces_enabled( + metadata + ): + return None + mode = forward_batch.forward_mode + if mode.is_decode(): + return 1 + if not mode.is_target_verify(): + return None + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + C4_ROWGROUP_MIN_VERIFY_ROWS, + logits_rows_per_chunk, + ) + + page_table = metadata.indexer_metadata.page_table + rows, num_draft = page_table.shape[0], self.target_verify_num_draft_tokens + if ( + num_draft < 2 + or rows != num_draft * forward_batch.batch_size + or rows < C4_ROWGROUP_MIN_VERIFY_ROWS + or rows > logits_rows_per_chunk(page_table) + ): + return None + return num_draft + def _refresh_fp4_prefill_workspace( self, forward_batch: ForwardBatch, metadata: Optional[DSV4Metadata] = None ) -> None: @@ -1775,6 +1860,13 @@ class DeepseekV4HipRadixBackend( refresh_low_ratio_prefill_workspaces( metadata.low_ratio_indexer_metadata_by_ratio(), metadata.fp4_low_ratio_prefill_workspaces, + extend_lens=( + metadata.late_layer_tail.extend_seq_lens_cpu + if metadata.late_layer_tail is not None + else _as_int_list(forward_batch.extend_seq_lens_cpu) + ), + max_context_len=self.max_context_len, + page8_ratios=self.token_to_kv_pool.low_ratio_index_k_page8_ratios(), ) ) if not self._fp4_workspaces_enabled(metadata): @@ -1785,6 +1877,9 @@ class DeepseekV4HipRadixBackend( ForwardMode.TARGET_VERIFY, ): return + if self._fp4_c4_rowgroup_rows(metadata, forward_batch) is not None: + # scored on the row-group kernel, its workspace built with the metadata + return if ( get_parallel().attn_cp_size != 1 or getattr(forward_batch, "tbo_children", None) diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index f11ec3f368..36b08e03c8 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -24,6 +24,7 @@ from sglang.kernels.ops.attention.dsv4 import ( topk_transform_paged_v2, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + FP4RowgroupDecodeWorkspace, aiter_fp4_paged_mqa_logits, aiter_q_indexer_fp4, logits_rows_per_chunk, @@ -1018,6 +1019,14 @@ class C4IndexerBackendMixin: elif use_aiter_fp4: q_fp4, q_scale = q is_decode = forward_batch.forward_mode.is_decode() + if isinstance( + getattr(metadata, "fp4_decode_workspace", None), + FP4RowgroupDecodeWorkspace, + ) and (is_decode or forward_batch.forward_mode.is_target_verify()): + # The backend built a row-group workspace for this step: target-verify + # rows score like decode rows, one launch over the whole batch, a + # request's rows sharing each key load. + is_decode = True # Hoisted: these await this layer's KV transfer, which every chunk # would otherwise re-await. k_payload = token_to_kv_pool.get_index_k_fp4_payload_buffer( diff --git a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py index 97e619a147..ac85618f3f 100644 --- a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py @@ -3,8 +3,19 @@ payload / scale index-K pools, then the top-k v2 transform -- the DeepGEMM path' from __future__ import annotations +import functools import itertools -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +import logging +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + FrozenSet, + List, + Optional, + Tuple, + Union, +) import torch import triton @@ -17,12 +28,17 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( slice_candidate_blocks, topk_transform_paged_hip, topk_within_candidate_blocks_hip, + topk_within_candidate_blocks_page8, + with_candidate_page8_table, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( LOW_RATIO_PAGE_TABLE_BUCKET, FP4DecodeWorkspace, FP4PrefillWorkspace, + FP4RowgroupDecodeWorkspace, + FP4RowgroupPrefillWorkspace, aiter_fp4_paged_mqa_logits, + aiter_has_fp4_rowgroup, index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, indexer_head_weights, @@ -30,11 +46,17 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( pack_fp4_query_flydsl, prepare_fp4_decode_workspace, prepare_fp4_prefill_workspace, + prepare_fp4_rowgroup_decode_workspace, + prepare_fp4_rowgroup_prefill_workspace, + rowgroup_paged_mqa_logits, + rowgroup_prefill_query_start_loc, + rowgroup_table_columns, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( index_k_norm_rope_pack_store_split, ) from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.srt.environ import envs from sglang.srt.layers.attention.deepseek_v4_backend import ( _as_int_list, ) @@ -47,6 +69,61 @@ if TYPE_CHECKING: # fp32 logits scored per candidate-block chunk stay under this many bytes _CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30 +logger = logging.getLogger(__name__) + + +@functools.lru_cache(maxsize=1) +def _fp4_rowgroup_enabled() -> bool: + """SGLANG_HIP_FP4_INDEXER_ROWGROUP, off when the installed AITER has no row-group + kernel (ROCm/aiter#6145).""" + if not envs.SGLANG_HIP_FP4_INDEXER_ROWGROUP.get(): + return False + if not aiter_has_fp4_rowgroup(): + logger.warning( + "SGLANG_HIP_FP4_INDEXER_ROWGROUP is set but the installed AITER has no " + "row-group FP4 MQA logits kernel (#6145); keeping the schedule kernels." + ) + return False + return True + + +def _score_requests_page8( + *, + q_fp4: torch.Tensor, + q_scale: torch.Tensor, + weights: torch.Tensor, + k_payload: torch.Tensor, + k_scale: torch.Tensor, + workspace: FP4RowgroupPrefillWorkspace, + rows: slice, + req_lo: int, + rows_per_request: List[int], + row_ends: torch.Tensor, +) -> torch.Tensor: + """Full-context logits of a page8 pool for rows, consecutive requests from req_lo on with + rows_per_request rows each; a request's rows share every key load.""" + assert sum(rows_per_request) == q_fp4.shape[0], (rows_per_request, q_fp4.shape) + return rowgroup_paged_mqa_logits( + q_fp4=q_fp4, + q_scale=q_scale, + k_payload=k_payload, + k_scale=k_scale, + weights=weights, + weight_scale=1.0, + block_tables=workspace.request_page_table[ + req_lo : req_lo + len(rows_per_request) + ], + query_start_loc=rowgroup_prefill_query_start_loc( + workspace, rows, rows_per_request + ), + row_ends=row_ends, + max_query_len=max(rows_per_request), + max_seq_len=workspace.max_seq_len, + page8=True, + pages_per_block=8, + is_decode=False, + ) + def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool: """Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch.""" @@ -114,33 +191,74 @@ def _indexer_inputs(layer, x, q_lora, pos): def build_low_ratio_decode_workspaces( metadata_by_ratio: Dict[int, PagedIndexerMetadata], -) -> Dict[int, FP4DecodeWorkspace]: - """Capture-safe: everything the schedule kernel touches is pinned in the workspace.""" - return { - ratio: prepare_fp4_decode_workspace( - meta.page_table, - meta.compressed_seq_lens, - page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + *, + max_context_len: int, + num_requests: Optional[int] = None, + page8_ratios: FrozenSet[int] = frozenset(), +) -> Dict[int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]]: + """Capture-safe: everything the schedule kernel touches is pinned in the workspace. + With the row-group kernel there is no schedule; a request's rows (``num_requests`` + requests of equal row counts, consecutive: decode 1, target-verify its draft rows) + score as one sequence, on a table as wide as ``max_context_len`` needs. A page8 pool + is read by the row-group kernel only.""" + rowgroup = _fp4_rowgroup_enabled() + workspaces = {} + for ratio, meta in metadata_by_ratio.items(): + if not (rowgroup or ratio in page8_ratios): + workspaces[ratio] = prepare_fp4_decode_workspace( + meta.page_table, + meta.compressed_seq_lens, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + ) + continue + rows = meta.page_table.shape[0] + rows_per_request = ( + rows // num_requests if num_requests and rows % num_requests == 0 else 1 ) - for ratio, meta in metadata_by_ratio.items() - } + workspaces[ratio] = prepare_fp4_rowgroup_decode_workspace( + meta.page_table, + rows_per_request, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + table_columns=rowgroup_table_columns( + max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET + ), + ) + return workspaces def refresh_low_ratio_prefill_workspaces( metadata_by_ratio: Dict[int, PagedIndexerMetadata], previous: Optional[Dict[int, FP4PrefillWorkspace]], -) -> Dict[int, FP4PrefillWorkspace]: - """Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace.""" + *, + extend_lens: Optional[List[int]], + max_context_len: int, + page8_ratios: FrozenSet[int] = frozenset(), +) -> Dict[int, Union[FP4PrefillWorkspace, FP4RowgroupPrefillWorkspace]]: + """Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace. A page8 pool + scores on the row-group kernel, which takes no schedule, on a table as wide as + ``max_context_len`` needs (without extend_lens, the indexer builds its workspace + itself).""" previous = previous or {} - return { - ratio: prepare_fp4_prefill_workspace( + workspaces = {} + for ratio, meta in metadata_by_ratio.items(): + if ratio in page8_ratios: + if extend_lens is not None: + workspaces[ratio] = prepare_fp4_rowgroup_prefill_workspace( + meta.page_table, + extend_lens, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + table_columns=rowgroup_table_columns( + max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET + ), + ) + continue + workspaces[ratio] = prepare_fp4_prefill_workspace( meta.page_table, meta.compressed_seq_lens, workspace=previous.get(ratio), page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, ) - for ratio, meta in metadata_by_ratio.items() - } + return workspaces def low_ratio_identity_skip_enabled( @@ -277,11 +395,37 @@ def low_ratio_index_topk_hip_decode( two_level = False q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos) + k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id) + k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id) + page8 = pool.low_ratio_index_k_page8(layer.layer_id) + if two_level and indexer.uses_candidates: + candidates = backend.candidate_masks + assert ( + isinstance(candidates, CandidateBlocks) + and candidates.ids.shape[0] == q_fp4.shape[0] + and candidates.block_size == indexer.candidate_block_size + ), "candidate blocks missing for decode" + if page8: + topk_within_candidate_blocks_page8( + q_fp4=q_fp4, + q_scale=q_scale, + weights=weights, + k_payload=k_payload, + k_scale=k_scale, + candidates=candidates, + seq_lens=indexer_metadata.compressed_seq_lens, + page_table=indexer_metadata.page_table, + page_size=indexer_metadata.compressed_page_size, + page_indices=core.sparse_page_indices(ratio), + raw_indices=core.sparse_raw_indices(ratio), + is_decode=True, + ) + return logits = aiter_fp4_paged_mqa_logits( q_fp4=q_fp4, q_scale=q_scale, - k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id), - k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id), + k_payload=k_payload, + k_scale=k_scale, weights=weights, page_table=indexer_metadata.page_table, c4_seq_lens=indexer_metadata.compressed_seq_lens, @@ -289,15 +433,13 @@ def low_ratio_index_topk_hip_decode( page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, is_decode=True, decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio), + page8=page8, + rowgroup_table_columns=rowgroup_table_columns( + backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET + ), ) # level one is bounded on device by the compressed lengths: a captured step cannot read them back if two_level and indexer.uses_candidates: - candidates = backend.candidate_masks - assert ( - isinstance(candidates, CandidateBlocks) - and candidates.ids.shape[0] == logits.shape[0] - and candidates.block_size == indexer.candidate_block_size - ), "candidate blocks missing for decode" topk_within_candidate_blocks_hip( logits, indexer_metadata.compressed_seq_lens, @@ -309,12 +451,20 @@ def low_ratio_index_topk_hip_decode( ) return if two_level and indexer.is_candidate_source: - backend.candidate_masks = select_candidate_blocks_hip( + candidates = select_candidate_blocks_hip( logits, indexer_metadata.compressed_seq_lens, topk_blocks=indexer.candidate_topk_blocks, block_size=indexer.candidate_block_size, ) + if page8: + candidates = with_candidate_page8_table( + candidates, + indexer_metadata.compressed_seq_lens, + indexer_metadata.page_table, + page_size=indexer_metadata.compressed_page_size, + ) + backend.candidate_masks = candidates topk_transform_paged_hip( logits, indexer_metadata.compressed_seq_lens, @@ -426,8 +576,36 @@ def low_ratio_index_topk_hip_extend( k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id) k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id) prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio) + page8 = pool.low_ratio_index_k_page8(layer.layer_id) + if page8 and not ( + isinstance(prefill_workspace, FP4RowgroupPrefillWorkspace) + and prefill_workspace.request_page_table.shape[0] == len(extend_lens_cpu) + ): + prefill_workspace = prepare_fp4_rowgroup_prefill_workspace( + indexer_metadata.page_table, + extend_lens_cpu, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + table_columns=rowgroup_table_columns( + backend.max_context_len, ratio, LOW_RATIO_PAGE_TABLE_BUCKET + ), + ) - def score_rows(rows: slice) -> torch.Tensor: + def score_rows( + rows: slice, req_lo: int, rows_per_request: List[int] + ) -> torch.Tensor: + if page8: + return _score_requests_page8( + q_fp4=q_fp4[rows], + q_scale=q_scale[rows], + weights=weights[rows], + k_payload=k_payload, + k_scale=k_scale, + workspace=prefill_workspace, + rows=rows, + req_lo=req_lo, + rows_per_request=rows_per_request, + row_ends=indexer_metadata.compressed_seq_lens[rows], + ) return aiter_fp4_paged_mqa_logits( q_fp4=q_fp4[rows], q_scale=q_scale[rows], @@ -442,16 +620,58 @@ def low_ratio_index_topk_hip_extend( prefill_workspace=prefill_workspace if rows.start == 0 else None, ) + def select_rows_page8_consumer(rows: slice, req_lo, rows_per_request, consume_rows): + """A consumer on a page8 pool scores only a request's candidate blocks.""" + + def full_logits(r: slice, b: int) -> torch.Tensor: + g = slice(rows.start + r.start, rows.start + r.stop) + return score_rows(g, req_lo + b, [r.stop - r.start]) + + def within(r: slice, candidates, rows_page, rows_raw) -> None: + g = slice(rows.start + r.start, rows.start + r.stop) + topk_within_candidate_blocks_page8( + q_fp4=q_fp4[g], + q_scale=q_scale[g], + weights=weights[g], + k_payload=k_payload, + k_scale=k_scale, + candidates=candidates, + seq_lens=compress_lens[g], + page_table=indexer_metadata.page_table[g], + page_size=indexer_metadata.compressed_page_size, + page_indices=rows_page, + raw_indices=rows_raw, + is_decode=False, + ) + + _consume_candidates_per_request( + extend_lens_cpu=rows_per_request, + consume=consume_rows, + compress_lens=compress_lens[rows], + page_table=indexer_metadata.page_table[rows], + page_size=indexer_metadata.compressed_page_size, + page_indices=page_indices[rows], + raw_indices=raw_indices[rows] if raw_indices is not None else None, + full_logits=full_logits, + within=within, + ) + def select_rows( rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish ): + rows_per_request = ( + [rows.stop - rows.start] + if req_hi == req_lo + 1 + else extend_lens_cpu[req_lo:req_hi] + ) + if page8 and consume_rows is not None: + select_rows_page8_consumer(rows, req_lo, rows_per_request, consume_rows) + return _select_topk_extend_hip( indexer=indexer, - logits=score_rows(rows), + logits=score_rows(rows, req_lo, rows_per_request), lc_per_req=lc_per_req[req_lo:req_hi], - extend_lens_cpu=[rows.stop - rows.start] - if req_hi == req_lo + 1 - else extend_lens_cpu[req_lo:req_hi], + extend_lens_cpu=rows_per_request, is_identity=group_is_identity, compress_lens=compress_lens[rows], page_table=indexer_metadata.page_table[rows], @@ -503,6 +723,19 @@ def low_ratio_index_topk_hip_extend( if publish is not None: publish.append(cat_candidate_blocks(pieces)) if publish is not None: + if page8: + assert len(publish) == len(extend_lens_cpu), (len(publish), extend_lens_cpu) + tok = 0 + for b, candidates in enumerate(publish): + r = slice(tok, tok + extend_lens_cpu[b]) + tok = r.stop + if candidates is not None: + publish[b] = with_candidate_page8_table( + candidates, + compress_lens[r], + indexer_metadata.page_table[r], + page_size=indexer_metadata.compressed_page_size, + ) backend.candidate_masks = publish @@ -551,33 +784,27 @@ def _select_topk_extend_hip( ) publish.append(cat_candidate_blocks(pieces)) if consume is not None: - tok_start = 0 - for b, t_len in enumerate(extend_lens_cpu): - rows = slice(tok_start, tok_start + t_len) - tok_start += t_len - if t_len == 0: - continue - rows_page = page_indices[rows] - rows_raw = raw_indices[rows] if raw_indices is not None else None - if consume[b] is None: - topk_transform_paged_hip( - logits[rows], - compress_lens[rows].contiguous(), - page_table[rows], - rows_page, - page_size, - rows_raw, - ) - else: + _consume_candidates_per_request( + extend_lens_cpu=extend_lens_cpu, + consume=consume, + compress_lens=compress_lens, + page_table=page_table, + page_size=page_size, + page_indices=page_indices, + raw_indices=raw_indices, + full_logits=lambda rows, b: logits[rows], + within=lambda rows, candidates, rows_page, rows_raw: ( topk_within_candidate_blocks_hip( logits[rows], compress_lens[rows], - consume[b], + candidates, page_table=page_table[rows], page_size=page_size, page_indices=rows_page, raw_indices=rows_raw, ) + ), + ) return topk_transform_paged_hip( logits, @@ -589,6 +816,43 @@ def _select_topk_extend_hip( ) +def _consume_candidates_per_request( + *, + extend_lens_cpu: List[int], + consume: List[Optional[CandidateBlocks]], + compress_lens: torch.Tensor, + page_table: torch.Tensor, + page_size: int, + page_indices: torch.Tensor, + raw_indices: Optional[torch.Tensor], + full_logits: Callable[[slice, int], torch.Tensor], + within: Callable[ + [slice, CandidateBlocks, torch.Tensor, Optional[torch.Tensor]], None + ], +) -> None: + """A consumer layer's rows, request by request: within(rows, candidates, ...) selects inside + the published blocks; a request with none runs the paged top-k of full_logits(rows, b).""" + tok_start = 0 + for b, t_len in enumerate(extend_lens_cpu): + rows = slice(tok_start, tok_start + t_len) + tok_start += t_len + if t_len == 0: + continue + rows_page = page_indices[rows] + rows_raw = raw_indices[rows] if raw_indices is not None else None + if consume[b] is None: + topk_transform_paged_hip( + full_logits(rows, b), + compress_lens[rows].contiguous(), + page_table[rows], + rows_page, + page_size, + rows_raw, + ) + else: + within(rows, consume[b], rows_page, rows_raw) + + def _request_groups( extend_lens_cpu: List[int], rows_per_chunk: int ) -> List[Tuple[int, int, int, int]]: @@ -620,4 +884,5 @@ def store_index_k_split( pool.get_index_k_fp4_payload_buffer(layer_id), pool.get_index_k_fp4_scale_buffer(layer_id), ratio=layer.compress_ratio, + page8=pool.low_ratio_index_k_page8(layer_id), ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index bcf09b82bf..59a9e74056 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -2,7 +2,16 @@ from __future__ import annotations import logging from contextlib import nullcontext -from typing import List, Literal, NamedTuple, Optional, Sequence, Tuple, Union +from typing import ( + FrozenSet, + List, + Literal, + NamedTuple, + Optional, + Sequence, + Tuple, + Union, +) import torch @@ -557,6 +566,10 @@ class DeepSeekV4IndexerPool(KVCache): self.uses_aiter_fp4_layout = _is_hip and self.use_fp4_indexer # Low-ratio pools round to nearest even; c4 keeps threshold rounding. self.index_k_rne = False + # aiter's row-group page-8 layout inside each page (pack_kv_cache): page size and shapes + # are unchanged but the bytes are not, so PD peers must use the same layout and a + # persisted cache written under the other one must be cleared before switching + self.index_k_page8 = False self._create_buffer() @@ -702,6 +715,7 @@ class DeepSeekV4IndexerPool(KVCache): loc, page_size=self.page_size, rne=self.index_k_rne, + page8=self.index_k_page8, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( store_fp4_index_k_cache, @@ -1533,6 +1547,19 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): kv_layout=self.compressed_kv_layout(ratio), ) + index_k_page8 = _is_hip and envs.SGLANG_HIP_DSV41_INDEX_K_PAGE8.get() + if index_k_page8: + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + aiter_has_fp4_rowgroup, + ) + + if not aiter_has_fp4_rowgroup(): + logger.warning( + "SGLANG_HIP_DSV41_INDEX_K_PAGE8 is set but the installed AITER has " + "no row-group FP4 MQA logits kernel (#6145); keeping the page-64 " + "index-K pool." + ) + index_k_page8 = False self.index_pools: dict[int, DeepSeekV4IndexerPool] = { ratio: self._make_indexer_pool( config.indexer_size, @@ -1559,6 +1586,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): device, enable_memory_saver, force_fp4=True, + # only the candidate consumers (ratio 1) score a subset of blocks + index_k_page8=ratio == 1 and index_k_page8, ) # HiCache and hardware backends still read these per-ratio attributes. @@ -1616,6 +1645,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): enable_memory_saver: bool, force_fp4: bool = False, global_page_size: Optional[int] = None, + index_k_page8: bool = False, ) -> DeepSeekV4IndexerPool: """Build the c4 lightning-indexer K pool (packed CUDA layout). Overridden by :class:`DSV4NPUTokenToKVPool` to swap in the @@ -1636,6 +1666,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): ) # The dsv41 low-ratio indexer rounds to nearest even (reference rounding). pool.index_k_rne = True + pool.index_k_page8 = index_k_page8 return pool return DeepSeekV4IndexerPool( size, @@ -1998,6 +2029,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): compress_ratio, _, _ = self.layer_mapping[layer_id] return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout + def low_ratio_index_k_page8(self, layer_id: int) -> bool: + """Whether the layer's split index-K pool lays each page out as row-group 8-slot pages.""" + compress_ratio, _, _ = self.layer_mapping[layer_id] + return self._indexer_pool(compress_ratio).index_k_page8 + + def low_ratio_index_k_page8_ratios(self) -> FrozenSet[int]: + """The compression ratios whose index-K pool is in the page-8 layout.""" + return frozenset( + ratio for ratio, pool in self.index_pools.items() if pool.index_k_page8 + ) + def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor: self.wait_layer_transfer(layer_id) compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id] diff --git a/test/registered/amd/test_dsv4_hip_bcg_metadata.py b/test/registered/amd/test_dsv4_hip_bcg_metadata.py index 7af434c8aa..fddd29a5d1 100644 --- a/test/registered/amd/test_dsv4_hip_bcg_metadata.py +++ b/test/registered/amd/test_dsv4_hip_bcg_metadata.py @@ -79,6 +79,7 @@ def _make_backend(*, block_size, device, is_dspark_draft=True, low_ratios=()): backend.is_dspark = True backend.needs_cpu_seq_lens = False backend._fp4_graph_row_limit = None + backend._fp4_c4_rowgroup_columns = None backend.is_draft_worker = is_dspark_draft # compressed-KV metadata is built only on the target worker backend.need_compress = not is_dspark_draft diff --git a/test/registered/kernels/benchmark/attention/bench_dsv4_c4_indexer_rowgroup_hip.py b/test/registered/kernels/benchmark/attention/bench_dsv4_c4_indexer_rowgroup_hip.py new file mode 100644 index 0000000000..d3469efef4 --- /dev/null +++ b/test/registered/kernels/benchmark/attention/bench_dsv4_c4_indexer_rowgroup_hip.py @@ -0,0 +1,240 @@ +"""DeepSeek-V4 C4 FP4 indexer logits on gfx950: today's schedule kernels against AITER's +row-group kernel (SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP) for decode (1 row a request) and +DSpark target-verify (4 or 7 rows a request). + +Shapes are the serving ones: 64 heads, 64-slot index-K pages, a page table as wide as the +captured graph's (``--max-context``, the server's context length), every request at the +same context. Each timed launch reads its own pages (``--rotate-mb`` of index K in all), so +the keys come from HBM as they do between layers. + +* ``sched_kern``: per layer, today's kernel with its workspace built: the varctx decode + kernel (decode) or the prefill kernel (verify). +* ``sched_ws``: per step, today's workspace build in the graph: the decode schedule or + the prefill schedule (DSpark verify builds it in the graph, upgraded from raw metadata). +* ``rg_grp`` / ``rg_row``: per layer, the row-group kernel, a request's rows one sequence / + each row its own (decode: one row a request either way, shown as ``rg_row``). +* ``rg_ws``: per step, the row-group workspace build (in the graph). +* ``step_old`` / ``step_new``: per step over ``--c4-layers`` C4 layers (DeepSeek-V4-Pro: + 30), workspace included; ``step_new`` is what the backend picks: decode on ``rg_row``, + verify on ``rg_grp`` from ``C4_ROWGROUP_MIN_VERIFY_ROWS`` rows, else today's path. + +Kernel and workspace times are CUDA-graph replays divided by the launches in them. +""" + +from __future__ import annotations + +import argparse + +import torch + +from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + C4_ROWGROUP_MIN_VERIFY_ROWS, + aiter_fp4_paged_mqa_logits, + prepare_fp4_decode_workspace, + prepare_fp4_prefill_workspace, + prepare_fp4_rowgroup_decode_workspace, + rowgroup_table_columns, +) +from sglang.test.ci.ci_register import register_amd_ci + +register_amd_ci( + est_time=300, + stage="jit-kernel-benchmark", + runner_config="amd", + disabled="manual microbenchmark", +) + +HEADS = 64 +HEAD_DIM = 128 +PAGE = 64 +RATIO = 4 +PAGE_BYTES = PAGE * HEAD_DIM // 2 + PAGE * HEAD_DIM // 32 + + +def _graph_time_us(fn, launches: int, iters: int) -> float: + fn() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + fn() + for _ in range(3): + graph.replay() + torch.cuda.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(iters): + graph.replay() + end.record() + torch.cuda.synchronize() + return start.elapsed_time(end) * 1e3 / iters / launches + + +def _case(bs: int, ctx: int, rows: int, max_context: int, rotate_mb: int): + dev = "cuda" + keys = ctx // RATIO + pages_per_seq = -(-keys // PAGE) + step_pages = bs * pages_per_seq + n_rot = max(1, min(32, rotate_mb * 2**20 // (step_pages * PAGE_BYTES))) + num_pages = n_rot * step_pages + payload = torch.randint( + 0, 256, (num_pages, 1, 4, PAGE, 16), dtype=torch.uint8, device=dev + ) + scale = torch.randint( + 110, 130, (num_pages, 1, 4, PAGE), dtype=torch.uint8, device=dev + ) + width = rowgroup_table_columns(max_context, RATIO) - 4 + n = bs * rows + pos = torch.tensor( + [ctx - rows + r for _ in range(bs) for r in range(rows)], device=dev + ) + row_ends = ((pos + 1) // RATIO).to(torch.int32) + steps = [] + perm = torch.randperm(num_pages, device=dev).to(torch.int32) + for i in range(n_rot): + table = torch.zeros(bs, width, dtype=torch.int32, device=dev) + table[:, :pages_per_seq] = perm[i * step_pages : (i + 1) * step_pages].view( + bs, pages_per_seq + ) + steps.append( + dict( + q_fp4=torch.randint( + 0, 256, (n, HEADS, HEAD_DIM // 2), dtype=torch.uint8, device=dev + ), + q_scale=torch.randint( + 120, 128, (n, 1, 4, 16, 4), dtype=torch.uint8, device=dev + ), + weights=torch.randn(n, HEADS, device=dev, dtype=torch.bfloat16), + page_table=table.repeat_interleave(rows, dim=0).contiguous(), + ) + ) + return payload, scale, row_ends, steps + + +def _logits(payload, scale, row_ends, step, is_decode, decode_ws=None, prefill_ws=None): + return aiter_fp4_paged_mqa_logits( + q_fp4=step["q_fp4"], + q_scale=step["q_scale"], + k_payload=payload, + k_scale=scale, + weights=step["weights"], + page_table=step["page_table"], + c4_seq_lens=row_ends, + weight_scale=0.01, + is_decode=is_decode, + decode_workspace=decode_ws, + prefill_workspace=prefill_ws, + ) + + +def bench(bs, ctx, rows, args): + payload, scale, row_ends, steps = _case( + bs, ctx, rows, args.max_context, args.rotate_mb + ) + columns = rowgroup_table_columns(args.max_context, RATIO) + launches = len(steps) + out = {} + + def rg_ws(step, **kw): + kw.setdefault("min_grouped_seqs", 1) + return prepare_fp4_rowgroup_decode_workspace( + step["page_table"], kw.pop("rows", rows), table_columns=columns, **kw + ) + + for name, kw in (("rg_grp", {}), ("rg_row", {"rows": 1})): + if name == "rg_grp" and rows == 1: + continue + wss = [rg_ws(s, **dict(kw)) for s in steps] + + def run(wss=wss): + for s, ws in zip(steps, wss): + _logits(payload, scale, row_ends, s, True, decode_ws=ws) + + out[name] = _graph_time_us(run, launches, args.iters) + ws_reps = 8 + + def rg_ws_build(): + for _ in range(ws_reps): + rg_ws(steps[0]) + + out["rg_ws"] = _graph_time_us(rg_ws_build, ws_reps, args.iters) + + if rows == 1: + wss = [prepare_fp4_decode_workspace(s["page_table"], row_ends) for s in steps] + + def run_sched(): + for s, ws in zip(steps, wss): + _logits(payload, scale, row_ends, s, True, decode_ws=ws) + + def sched_ws_build(): + for _ in range(ws_reps): + prepare_fp4_decode_workspace(steps[0]["page_table"], row_ends) + + out["sched_ws"] = _graph_time_us(sched_ws_build, ws_reps, args.iters) + else: + wss = [prepare_fp4_prefill_workspace(s["page_table"], row_ends) for s in steps] + + def run_sched(): + for s, ws in zip(steps, wss): + _logits(payload, scale, row_ends, s, False, prefill_ws=ws) + + ws0 = wss[0] + + def sched_ws_build(): + for _ in range(ws_reps): + prepare_fp4_prefill_workspace( + steps[0]["page_table"], row_ends, workspace=ws0 + ) + + out["sched_ws"] = _graph_time_us(sched_ws_build, ws_reps, args.iters) + out["sched_kern"] = _graph_time_us(run_sched, launches, args.iters) + layers = args.c4_layers + out["step_old"] = layers * out["sched_kern"] + out["sched_ws"] + if rows == 1: + out["step_new"] = layers * out["rg_row"] + out["rg_ws"] + elif bs * rows >= C4_ROWGROUP_MIN_VERIFY_ROWS: + out["step_new"] = layers * out["rg_grp"] + out["rg_ws"] + else: + out["step_new"] = out["step_old"] + return out + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--batches", default="1,2,4,8,16,32,48") + parser.add_argument("--ctx", default="8192,32768,65536,131072,200000") + parser.add_argument("--rows", default="1,4,7") + parser.add_argument("--max-context", type=int, default=1 << 20) + parser.add_argument("--rotate-mb", type=int, default=1024) + parser.add_argument("--iters", type=int, default=20) + parser.add_argument("--c4-layers", type=int, default=30) + args = parser.parse_args() + cols = ( + "sched_kern", + "rg_grp", + "rg_row", + "sched_ws", + "rg_ws", + "step_old", + "step_new", + ) + print( + f"min_verify_rows={C4_ROWGROUP_MIN_VERIFY_ROWS} max_context={args.max_context}" + ) + print("| rows | bs | ctx | " + " | ".join(cols) + " | step x |") + print("|---" * (len(cols) + 4) + "|") + for rows in map(int, args.rows.split(",")): + for bs in map(int, args.batches.split(",")): + for ctx in map(int, args.ctx.split(",")): + r = bench(bs, ctx, rows, args) + cells = " | ".join(f"{r[c]:.1f}" if c in r else "-" for c in cols) + print( + f"| {rows} | {bs} | {ctx} | {cells} | " + f"{r['step_old'] / r['step_new']:.3f} |", + flush=True, + ) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py index e6db184b4e..8a52a1b680 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py @@ -27,19 +27,31 @@ from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, compress_norm_rope_store, ) +from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( + candidate_page8_table, + select_candidate_blocks_hip, + slice_candidate_blocks, + topk_within_candidate_blocks_hip, + topk_within_candidate_blocks_page8, + with_candidate_page8_table, +) from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( _fp4_e2m1_code_rne, quantize_fp4_indexer_tensor, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + LOW_RATIO_PAGE_TABLE_BUCKET, FP4KWriteMetadata, + FP4RowgroupDecodeWorkspace, _decode_cta_count, _fp4_e2m1_code_rne_lean, _guard_page_table, aiter_fp4_paged_mqa_logits, + aiter_has_fp4_rowgroup, aiter_k_indexer_fp4_cache_write, aiter_q_indexer_fp4, + index_k_page8_views, index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, indexer_head_weights, @@ -47,7 +59,12 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( prepare_fp4_decode_workspace, prepare_fp4_k_write_metadata, prepare_fp4_prefill_workspace, + prepare_fp4_rowgroup_decode_workspace, + prepare_fp4_rowgroup_prefill_workspace, read_fp4_index_k_split, + rowgroup_paged_mqa_logits, + rowgroup_prefill_query_start_loc, + rowgroup_table_columns, store_fp4_index_k_cache_split, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( @@ -59,6 +76,10 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( rope_tail_fake_quant_fp4, ) +from sglang.kernels.ops.attention.dsv4.topk import ( + plan_topk_v2, + topk_transform_paged_v2, +) from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k from sglang.srt.utils import get_device, is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci @@ -70,6 +91,12 @@ pytestmark = pytest.mark.skipif( reason="The FP4 indexer adapters wrap AITER CDNA4 (gfx95x) kernels.", ) +# The row-group kernels (8- and 64-row pages) need AITER >= ROCm/aiter#6145. +requires_aiter_rowgroup = pytest.mark.skipif( + not (is_hip() and is_gfx95_supported() and aiter_has_fp4_rowgroup()), + reason="AITER has no row-group FP4 MQA logits kernel (ROCm/aiter#6145).", +) + HEAD_DIM = 128 FP4_DIM = HEAD_DIM // 2 GROUP_SIZE = 32 @@ -920,6 +947,733 @@ def test_prefill_paged_mqa_logits(batch: int, seq_len: int) -> None: _assert_logits_agree(logits, case) +def _verify_rows_case(case, rows_per_request: int): + """Each request's row as the last ``rows_per_request`` causal rows of a target-verify + step: page table, Q and weights repeated per row, bounds ramping up to the context.""" + q = rows_per_request + offs = torch.arange(q - 1, -1, -1, device=get_device(), dtype=torch.int32) + row_ends = (case["context"][:, None] - offs[None, :]).clamp_min(0).reshape(-1) + + def rep(t): + return t.repeat_interleave(q, dim=0).contiguous() + + return { + **case, + "q_fp4": rep(case["q_fp4"].view(torch.uint8)).view(case["q_fp4"].dtype), + "q_scale": rep(case["q_scale"]), + "weights": rep(case["weights"]), + "page_table": rep(case["page_table"]), + "c4_seq_lens": row_ends.contiguous(), + "context": row_ends.contiguous(), + "ref_logits_fp4": rep(case["ref_logits_fp4"]), + "ref_logits_bf16": rep(case["ref_logits_bf16"]), + } + + +@requires_aiter_rowgroup +@pytest.mark.parametrize( + "batch,seq_len,rows_per_request", + [(1, 256, 1), (3, 512, 6), (4, 512, 6), (6, 1024, 6), (8, 2048, 1)], +) +def test_rowgroup_decode_logits_match_varctx( + batch: int, seq_len: int, rows_per_request: int +) -> None: + """The row-group decode path agrees with the varctx kernel within tolerance and picks the + same top-k; a request's rows scored as one sequence or one a row are bit for bit equal.""" + torch.manual_seed(batch * 300 + seq_len + rows_per_request) + case = _verify_rows_case( + _build_logits_case(batch, seq_len, shuffle_pages=True), rows_per_request + ) + varctx = _run_logits( + case, + is_decode=True, + decode_ws=prepare_fp4_decode_workspace(case["page_table"], case["c4_seq_lens"]), + ) + grouped_ws = prepare_fp4_rowgroup_decode_workspace( + case["page_table"], rows_per_request + ) + per_row_ws = prepare_fp4_rowgroup_decode_workspace(case["page_table"], 1) + assert isinstance(grouped_ws, FP4RowgroupDecodeWorkspace) + assert grouped_ws.max_seq_len == ( + prepare_fp4_decode_workspace( + case["page_table"], case["c4_seq_lens"] + ).max_seq_len + ) + grouped = _run_logits(case, is_decode=True, decode_ws=grouped_ws) + per_row = _run_logits(case, is_decode=True, decode_ws=per_row_ws) + + for row, ctx in enumerate(case["context"].tolist()): + if ctx == 0: + continue + assert torch.equal(grouped[row, :ctx], per_row[row, :ctx]), f"row {row}" + torch.testing.assert_close( + grouped[row, :ctx], + case["ref_logits_fp4"][row, :ctx], + rtol=2.0e-3, + atol=2.0e-3, + ) + torch.testing.assert_close( + grouped[row, :ctx], varctx[row, :ctx], rtol=1e-5, atol=1e-6 + ) + k = min(64, ctx) + assert torch.equal( + grouped[row, :ctx].topk(k).indices.sort().values, + varctx[row, :ctx].topk(k).indices.sort().values, + ), f"row {row} top-{k}" + + +def _page8_copy(payload, scale): + """The same K cache in the page8 layout (shapes and bytes per 64-slot page unchanged).""" + from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4_rowgroup import ( + pack_kv_cache, + ) + + slots = torch.arange(payload.shape[0] * PAGE_SIZE, device=get_device()) + packed, exps = _read_index_k_cache(payload, scale, slots) + ref_kv, ref_ks = pack_kv_cache(packed, exps.contiguous(), page_size=8) + payload8 = torch.zeros_like(payload.view(torch.uint8)).view(payload.dtype) + scale8 = torch.zeros_like(scale) + kv8, ks8 = index_k_page8_views(payload8, scale8) + kv8.view(-1).copy_(ref_kv.reshape(-1)) + ks8.view(-1).copy_(ref_ks.reshape(-1)) + return payload8, scale8 + + +def _rowgroup_logits(case, rows_per_request: int, *, page8: bool, is_decode: bool): + block_tables, max_seq_len = _guard_page_table( + case["page_table"][::rows_per_request].contiguous() + ) + rows = case["q_fp4"].shape[0] + return rowgroup_paged_mqa_logits( + q_fp4=case["q_fp4"], + q_scale=case["q_scale"], + k_payload=case["payload8"] if page8 else case["payload"], + k_scale=case["scale8"] if page8 else case["scale"], + weights=case["weights"], + weight_scale=case["weight_scale"], + block_tables=block_tables, + query_start_loc=torch.arange( + 0, rows + 1, rows_per_request, dtype=torch.int32, device=get_device() + ), + row_ends=case["c4_seq_lens"], + max_query_len=rows_per_request, + max_seq_len=max_seq_len, + page8=page8, + pages_per_block=8 if page8 else 1, + is_decode=is_decode, + ) + + +def _page8_case(batch, seq_len, rows_per_request, *, ctx_lens=None): + case = _verify_rows_case( + _build_logits_case(batch, seq_len, ctx_lens=ctx_lens, shuffle_pages=True), + rows_per_request, + ) + case["payload8"], case["scale8"] = _page8_copy(case["payload"], case["scale"]) + return case + + +@requires_aiter_rowgroup +@pytest.mark.parametrize("is_decode", [True, False]) +@pytest.mark.parametrize( + "batch,seq_len,rows_per_request,ctx_lens", + [ + (1, 256, 1, None), + (3, 512, 6, [512, 77, 300]), + (6, 1024, 6, None), + (8, 2048, 1, [2048, 1, 9, 1000, 2047, 64, 65, 1500]), + ], +) +def test_page8_full_scoring_matches_page64( + batch, seq_len, rows_per_request, ctx_lens, is_decode +) -> None: + """The page8 pool scored through the 64-slot page table (pages_per_block=8) gives the + page-64 pool's logits: same keys, same rows; only the walk granularity differs.""" + torch.manual_seed(batch * 400 + seq_len + rows_per_request) + case = _page8_case(batch, seq_len, rows_per_request, ctx_lens=ctx_lens) + ref = _rowgroup_logits(case, rows_per_request, page8=False, is_decode=is_decode) + got = _rowgroup_logits(case, rows_per_request, page8=True, is_decode=is_decode) + for row, ctx in enumerate(case["context"].tolist()): + if ctx == 0: + continue + torch.testing.assert_close( + got[row, :ctx], ref[row, :ctx], rtol=1e-5, atol=1e-6, msg=f"row {row}" + ) + torch.testing.assert_close( + got[row, :ctx], + case["ref_logits_fp4"][row, :ctx], + rtol=2.0e-3, + atol=2.0e-3, + ) + + +@requires_aiter_rowgroup +def test_rowgroup_prefill_workspace_scores_ragged_requests() -> None: + """A page8 prefill workspace scores consecutive requests' rows (one request empty) as the + page-64 pool scores every row alone, and reuses a row range's query_start_loc.""" + torch.manual_seed(13) + lens = [4, 0, 6] + case = _build_logits_case(3, 512, ctx_lens=[512, 300, 77], shuffle_pages=True) + case["payload8"], case["scale8"] = _page8_copy(case["payload"], case["scale"]) + counts = torch.tensor(lens, device=get_device()) + starts = torch.tensor([0, 4, 4], device=get_device()) + local = torch.arange(sum(lens), device=get_device()) - starts.repeat_interleave( + counts + ) + row_ends = ( + case["context"].repeat_interleave(counts) + - (counts.repeat_interleave(counts) - 1 - local) + ).clamp_min(0) + row_ends = row_ends.to(torch.int32).contiguous() + + def rep(t): + return t.repeat_interleave(counts, dim=0).contiguous() + + q_fp4 = rep(case["q_fp4"].view(torch.uint8)).view(case["q_fp4"].dtype) + q_scale, weights = rep(case["q_scale"]), rep(case["weights"]) + page_table = rep(case["page_table"]) + rows = slice(0, sum(lens)) + + workspace = prepare_fp4_rowgroup_prefill_workspace(page_table, lens) + query_start_loc = rowgroup_prefill_query_start_loc(workspace, rows, lens) + assert rowgroup_prefill_query_start_loc(workspace, rows, lens) is query_start_loc + assert query_start_loc.tolist() == [0, 4, 4, 10] + common = dict( + q_fp4=q_fp4, + q_scale=q_scale, + weights=weights, + weight_scale=case["weight_scale"], + row_ends=row_ends, + is_decode=False, + ) + got = rowgroup_paged_mqa_logits( + k_payload=case["payload8"], + k_scale=case["scale8"], + block_tables=workspace.request_page_table, + query_start_loc=query_start_loc, + max_query_len=max(lens), + max_seq_len=workspace.max_seq_len, + page8=True, + pages_per_block=8, + **common, + ) + per_row_table, max_seq_len = _guard_page_table(page_table) + ref = rowgroup_paged_mqa_logits( + k_payload=case["payload"], + k_scale=case["scale"], + block_tables=per_row_table, + query_start_loc=torch.arange( + sum(lens) + 1, dtype=torch.int32, device=get_device() + ), + max_query_len=1, + max_seq_len=max_seq_len, + page8=False, + **common, + ) + for row, end in enumerate(row_ends.tolist()): + torch.testing.assert_close( + got[row, :end], ref[row, :end], rtol=1e-5, atol=1e-6, msg=f"row {row}" + ) + + +@requires_aiter_rowgroup +def test_rowgroup_table_columns_fix_the_compiled_width() -> None: + """Workspaces built with rowgroup_table_columns share one block-table width whatever + the context, so the row-group kernel compiles one variant for it, and score exactly + as tables at their own width; the logits width stays each step's own.""" + torch.manual_seed(14) + columns = rowgroup_table_columns(1 << 20, 1, LOW_RATIO_PAGE_TABLE_BUCKET) + widths = set() + for ctx in (300, 5000, 9000): + case = _build_logits_case(1, ctx, ctx_lens=[ctx], shuffle_pages=True) + case["payload8"], case["scale8"] = _page8_copy(case["payload"], case["scale"]) + rows = 3 + row_ends = torch.arange(ctx - rows + 1, ctx + 1, dtype=torch.int32).to( + get_device() + ) + page_table = case["page_table"].expand(rows, -1).contiguous() + q_fp4 = case["q_fp4"].view(torch.uint8).expand(rows, -1, -1).contiguous() + common = dict( + q_fp4=q_fp4.view(case["q_fp4"].dtype), + q_scale=case["q_scale"].expand(rows, -1, -1, -1, -1).contiguous(), + weights=case["weights"].expand(rows, -1).contiguous(), + k_payload=case["payload8"], + k_scale=case["scale8"], + weight_scale=case["weight_scale"], + query_start_loc=torch.tensor( + [0, rows], dtype=torch.int32, device=get_device() + ), + row_ends=row_ends, + max_query_len=rows, + page8=True, + pages_per_block=8, + is_decode=False, + ) + fixed = prepare_fp4_rowgroup_prefill_workspace( + page_table, [rows], LOW_RATIO_PAGE_TABLE_BUCKET, table_columns=columns + ) + own = prepare_fp4_rowgroup_prefill_workspace( + page_table, [rows], LOW_RATIO_PAGE_TABLE_BUCKET + ) + assert fixed.request_page_table.shape[1] == columns + assert fixed.max_seq_len == own.max_seq_len + widths.add(fixed.request_page_table.stride(0)) + got = rowgroup_paged_mqa_logits( + block_tables=fixed.request_page_table, + max_seq_len=fixed.max_seq_len, + **common, + ).clone() + ref = rowgroup_paged_mqa_logits( + block_tables=own.request_page_table, max_seq_len=own.max_seq_len, **common + ) + for row, end in enumerate(row_ends.tolist()): + assert torch.equal(got[row, :end], ref[row, :end]), f"{ctx=} row {row}" + assert widths == {columns} + + +def _selected_slots(page_indices: torch.Tensor) -> list[list[int]]: + return [sorted(v for v in row if v >= 0) for row in page_indices.tolist()] + + +# --------------------------------------------------------------------------- +# DeepSeek-V4 C4 decode / DSpark target-verify on the row-group kernel +# (SGLANG_HIP_DSV4_C4_INDEXER_ROWGROUP): the backend builds a row-group decode +# workspace for both modes and the indexer scores verify rows as decode rows. +# --------------------------------------------------------------------------- + +_C4_RATIO = 4 +# DeepSeek-V4-Pro's index_topk +_C4_TOPK = 1024 +# The server's context length the C4 table width is fixed for; small enough for a test. +_C4_MAX_CONTEXT = 1 << 18 + + +def _c4_step_case(seq_lens: list[int], rows_per_request: int): + """One DSv4 C4 indexer step: a decode (one row a request) or a DSpark target-verify + (rows_per_request causal rows a request, each with its own query). ``seq_lens`` are + the requests' token lengths after the step, so row r of request b sits at token + position seq_lens[b] - rows_per_request + r and sees (pos + 1) // 4 compressed keys, + as c4_topk_lengths_raw has it. The index-K pages are shuffled and every byte the step + must not read is garbage; the page table is the indexer's: one row per query row, as + wide as the backend's graph table, zero past the request's pages.""" + q_rows = rows_per_request + batch = len(seq_lens) + dev = get_device() + keys = [-(-s // _C4_RATIO) for s in seq_lens] + pages_per_seq = -(-max(keys) // PAGE_SIZE) + padded_len = pages_per_seq * PAGE_SIZE + num_pages = batch * pages_per_seq + 5 + # queries rotate at token positions, keys at compressed ones + cos, sin, _ = _rope_tables(max(max(seq_lens), padded_len, 256)) + payload, scale = _empty_index_k_cache(num_pages) + payload.view(torch.uint8).random_(0, 256) + scale.random_(100, 140) + + physical = torch.randperm(num_pages, device=dev)[: batch * pages_per_seq] + req_pages = physical.to(torch.int32).reshape(batch, pages_per_seq) + kv_positions = torch.arange(padded_len, device=dev).repeat(batch) + loc = ( + req_pages.long()[:, :, None] * PAGE_SIZE + + torch.arange(PAGE_SIZE, device=dev)[None, None, :] + ).reshape(-1) + k = torch.randn(batch * padded_len, HEAD_DIM, device=dev, dtype=torch.bfloat16) + norm_weight = torch.randn(HEAD_DIM, device=dev, dtype=torch.bfloat16) + _write_index_k_cache(k, norm_weight, cos, sin, kv_positions, loc, payload, scale) + k_dq = _ref_dequantize_fp4_indexer(*_read_index_k_cache(payload, scale, loc)) + k_dq = k_dq.reshape(batch, padded_len, HEAD_DIM) + + pos = torch.tensor( + [s - q_rows + r for s in seq_lens for r in range(q_rows)], device=dev + ).clamp_min(0) + row_ends = ((pos + 1) // _C4_RATIO).to(torch.int32) + width = rowgroup_table_columns(_C4_MAX_CONTEXT, _C4_RATIO) - 4 + page_table = torch.zeros(batch, width, dtype=torch.int32, device=dev) + page_table[:, :pages_per_seq] = req_pages + page_table = page_table.repeat_interleave(q_rows, dim=0).contiguous() + + q = torch.randn( + batch * q_rows, NUM_HEADS, HEAD_DIM, device=dev, dtype=torch.bfloat16 + ) + q_fp4, q_scale = aiter_q_indexer_fp4(q.contiguous(), cos, sin, pos) + q_dq = _ref_dequantize_fp4_indexer( + q_fp4.view(torch.uint8), + _ref_quantize_fp4_indexer(_ref_q_transform(q, cos, sin, pos))[1], + ).reshape(batch, q_rows, NUM_HEADS, HEAD_DIM) + weights = torch.randn(batch * q_rows, NUM_HEADS, device=dev, dtype=torch.bfloat16) + weight_scale = HEAD_DIM**-0.5 * NUM_HEADS**-0.5 + per_head = torch.einsum("bqhd,bsd->bqhs", q_dq.float(), k_dq.float()).relu() + ref = weight_scale * torch.einsum( + "bqhs,bqh->bqs", per_head, weights.float().view(batch, q_rows, NUM_HEADS) + ) + return { + "q_fp4": q_fp4, + "q_scale": q_scale, + "payload": payload, + "scale": scale, + "weights": weights, + "weight_scale": weight_scale, + "page_table": page_table, + "c4_seq_lens": row_ends, + "context": row_ends, + "ref_logits_fp4": ref.reshape(batch * q_rows, padded_len), + "rows_per_request": q_rows, + } + + +def _c4_rowgroup_ws(case, rows_per_request=None, **kwargs): + """The workspace the backend builds for a C4 decode / target-verify step: a request's + rows one sequence.""" + kwargs.setdefault("min_grouped_seqs", 1) + return prepare_fp4_rowgroup_decode_workspace( + case["page_table"], + rows_per_request or case["rows_per_request"], + table_columns=rowgroup_table_columns(_C4_MAX_CONTEXT, _C4_RATIO), + **kwargs, + ) + + +def _c4_schedule_logits(case): + """Today's path: decode on the varctx kernel, target-verify on the prefill kernel.""" + if case["rows_per_request"] == 1: + ws = prepare_fp4_decode_workspace(case["page_table"], case["c4_seq_lens"]) + return _run_logits(case, is_decode=True, decode_ws=ws) + ws = prepare_fp4_prefill_workspace(case["page_table"], case["c4_seq_lens"]) + return _run_logits(case, is_decode=False, prefill_ws=ws) + + +def _c4_topk(case, logits): + """The production paged top-k (v2, index_topk 1024) over each row's own [0, len): + (page-table slots, raw positions).""" + rows = logits.shape[0] + pages = torch.empty(rows, _C4_TOPK, dtype=torch.int32, device=logits.device) + raw = torch.empty_like(pages) + lens = case["c4_seq_lens"] + topk_transform_paged_v2( + logits, lens, case["page_table"], pages, PAGE_SIZE, plan_topk_v2(lens), raw + ) + return pages, raw + + +def _assert_same_topk(case, raw_a, raw_b, scores, *, what: str) -> None: + """Equal top-k sets, up to keys tied with the k-th score within fp32 noise.""" + for row, end in enumerate(case["c4_seq_lens"].tolist()): + a = {int(v) for v in raw_a[row].tolist() if v >= 0} + b = {int(v) for v in raw_b[row].tolist() if v >= 0} + assert len(a) == len(b) == min(end, _C4_TOPK), (what, row, len(a), len(b)) + assert all(v < end for v in a | b), (what, row) + diff = a ^ b + if not diff: + continue + row_scores = scores[row, :end] + kth = row_scores.topk(min(end, _C4_TOPK)).values[-1] + near = (row_scores[list(diff)] - kth).abs() <= 1e-4 * kth.abs().clamp_min(1) + assert bool(near.all()), f"{what} row {row}: top-k differs off the tie {diff}" + + +_C4_STEP_CASES = [ + # decode: one row a request + ([9000], 1), + ([3000, 17000, 700], 1), + # DSpark block 3: 4 rows a request (DP8 c192+, TP c48) + ([9000], 4), + ([2500, 13000, 6000], 4), + ([16000, 2049, 8001, 600, 12345], 4), + # DSpark block 6: 7 rows a request (TP c4~c32) + ([12000], 7), + ([5000, 15000, 2100, 9100, 3, 7777], 7), +] + + +@requires_aiter_rowgroup +@pytest.mark.parametrize("seq_lens,rows_per_request", _C4_STEP_CASES) +def test_c4_rowgroup_matches_schedule_path_and_reference( + seq_lens, rows_per_request, monkeypatch +) -> None: + """A C4 decode / target-verify step on the row-group kernel scores what today's + kernels score, against the FP4 reference, and the top-k (v2, 1024) picks the same + keys; grouped (a request's rows one sequence) and per-row scoring agree bit for bit. + The row-group logits are poisoned past each row's bound, which the top-k never reads. + """ + torch.manual_seed(sum(seq_lens) + rows_per_request) + case = _c4_step_case(seq_lens, rows_per_request) + baseline = _c4_schedule_logits(case).clone() + + import sglang.kernels.ops.attention.dsv4.fp4_indexer_hip as hip_mod + + real_alloc = hip_mod._alloc_logits + + def poisoned(num_tokens, max_seq_len, device, is_decode): + return real_alloc(num_tokens, max_seq_len, device, is_decode).fill_( + float("inf") + ) + + monkeypatch.setattr(hip_mod, "_alloc_logits", poisoned) + ws = _c4_rowgroup_ws(case) + per_row_ws = _c4_rowgroup_ws(case, rows_per_request=1) + assert ws.guarded_page_table.shape[1] == rowgroup_table_columns( + _C4_MAX_CONTEXT, _C4_RATIO + ) + assert ws.rows_per_seq == rows_per_request + assert per_row_ws.rows_per_seq == 1 + got = _run_logits(case, is_decode=True, decode_ws=ws).clone() + per_row = _run_logits(case, is_decode=True, decode_ws=per_row_ws).clone() + + for row, end in enumerate(case["c4_seq_lens"].tolist()): + if end == 0: + continue + assert torch.equal(got[row, :end], per_row[row, :end]), f"row {row}" + torch.testing.assert_close( + got[row, :end], + case["ref_logits_fp4"][row, :end], + rtol=2.0e-3, + atol=2.0e-3, + msg=f"row {row} vs reference", + ) + torch.testing.assert_close( + got[row, :end], + baseline[row, :end], + rtol=1e-5, + atol=1e-6, + msg=f"row {row} vs schedule path", + ) + + pages_new, raw_new = _c4_topk(case, got) + pages_old, raw_old = _c4_topk(case, baseline) + _assert_same_topk(case, raw_new, raw_old, baseline, what="vs schedule path") + ref = case["ref_logits_fp4"] + _, raw_ref = _c4_topk(case, ref.contiguous()) + _assert_same_topk(case, raw_new, raw_ref, ref, what="vs reference") + # the paged output is the raw pick through the request's page table + sel = raw_new.clamp_min(0).long() + expect = case["page_table"].long().gather(1, sel // PAGE_SIZE) * PAGE_SIZE + ( + sel % PAGE_SIZE + ) + expect = torch.where(raw_new >= 0, expect, torch.full_like(expect, -1)) + assert torch.equal(pages_new.long(), expect) + + +@requires_aiter_rowgroup +def test_c4_rowgroup_stale_workspace_stays_on_rowgroup() -> None: + """A row-group workspace sized for another row count (DP padding, truncated + activations) is rebuilt per row on the same table width, not dropped onto the + schedule kernels.""" + torch.manual_seed(21) + case = _c4_step_case([6000, 9000, 4000, 7000], 4) + stale = _c4_rowgroup_ws(case) + rows = case["page_table"].shape[0] - 3 + sub = { + **case, + **{ + key: case[key][:rows] + for key in ("q_fp4", "q_scale", "weights", "page_table", "c4_seq_lens") + }, + } + with_stale = _run_logits(sub, is_decode=True, decode_ws=stale) + fresh = _run_logits( + sub, is_decode=True, decode_ws=_c4_rowgroup_ws(sub, rows_per_request=1) + ) + for row, end in enumerate(sub["c4_seq_lens"].tolist()): + assert torch.equal(with_stale[row, :end], fresh[row, :end]), f"row {row}" + + +@requires_aiter_rowgroup +@pytest.mark.parametrize("rows_per_request", [4, 7]) +def test_c4_rowgroup_workspace_and_logits_replay_in_a_graph( + rows_per_request: int, +) -> None: + """The backend builds the workspace inside the captured graph: a replay over new + lengths and pages scores as an eager step does.""" + torch.manual_seed(31 + rows_per_request) + first = _c4_step_case([8000, 3000, 12000, 5000, 9000], rows_per_request) + second = _c4_step_case([2000, 11000, 6100, 9999, 300], rows_per_request) + keys = ("q_fp4", "q_scale", "weights", "page_table", "c4_seq_lens") + static = {key: torch.empty_like(first[key]) for key in keys} + pool_pages = max(first["payload"].shape[0], second["payload"].shape[0]) + payload, scale = _empty_index_k_cache(pool_pages) + + def load(case): + for key in keys: + # byte views: the fp4 operands have no copy kernel of their own + static[key].view(torch.uint8).copy_(case[key].view(torch.uint8)) + payload.view(torch.uint8)[: case["payload"].shape[0]].copy_( + case["payload"].view(torch.uint8) + ) + scale[: case["scale"].shape[0]].copy_(case["scale"]) + + def step(): + ws = prepare_fp4_rowgroup_decode_workspace( + static["page_table"], + rows_per_request, + table_columns=rowgroup_table_columns(_C4_MAX_CONTEXT, _C4_RATIO), + min_grouped_seqs=1, + ) + return aiter_fp4_paged_mqa_logits( + q_fp4=static["q_fp4"], + q_scale=static["q_scale"], + k_payload=payload, + k_scale=scale, + weights=static["weights"], + page_table=static["page_table"], + c4_seq_lens=static["c4_seq_lens"], + weight_scale=first["weight_scale"], + is_decode=True, + decode_workspace=ws, + ) + + load(first) + step() # compile outside the capture + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = step() + for case in (second, first): + load(case) + graph.replay() + eager = step() + torch.cuda.synchronize() + for row, end in enumerate(case["c4_seq_lens"].tolist()): + assert torch.equal(out[row, :end], eager[row, :end]), f"row {row}" + torch.testing.assert_close( + out[row, :end], + case["ref_logits_fp4"][row, :end], + rtol=2.0e-3, + atol=2.0e-3, + ) + + +@requires_aiter_rowgroup +@pytest.mark.parametrize( + "batch,seq_len,rows_per_request,topk_blocks,topk", + [(2, 1024, 1, 32, 64), (3, 2048, 6, 64, 128), (4, 4096, 2, 128, 256)], +) +def test_page8_candidates_match_gather( + batch, seq_len, rows_per_request, topk_blocks, topk +) -> None: + """A consumer scoring only its candidate blocks on the page8 pool selects what top-k over + the full context's logits gathered to those blocks selects.""" + torch.manual_seed(batch * 500 + seq_len) + ctx = [seq_len - 3 * b for b in range(batch)] + case = _page8_case(batch, seq_len, rows_per_request, ctx_lens=ctx) + seq_lens = case["c4_seq_lens"] + full = _rowgroup_logits(case, rows_per_request, page8=False, is_decode=True) + candidates = select_candidate_blocks_hip( + full, seq_lens, topk_blocks=topk_blocks, block_size=8 + ) + rows = full.shape[0] + + def run(page8: bool): + page_indices = torch.full( + (rows, topk), -7, dtype=torch.int32, device=get_device() + ) + raw = torch.full_like(page_indices, -7) + common = dict( + page_table=case["page_table"], + page_size=PAGE_SIZE, + page_indices=page_indices, + raw_indices=raw, + ) + if page8: + topk_within_candidate_blocks_page8( + q_fp4=case["q_fp4"], + q_scale=case["q_scale"], + weights=case["weights"], + k_payload=case["payload8"], + k_scale=case["scale8"], + candidates=candidates, + seq_lens=seq_lens, + **common, + ) + else: + topk_within_candidate_blocks_hip(full, seq_lens, candidates, **common) + return page_indices, raw + + ref_pages, ref_raw = run(False) + got_pages, got_raw = run(True) + assert _selected_slots(got_pages) == _selected_slots(ref_pages) + assert _selected_slots(got_raw) == _selected_slots(ref_raw) + for row in range(rows): + n = int((ref_pages[row] >= 0).sum()) + assert (got_pages[row, :n] >= 0).all() and (got_pages[row, n:] == -1).all() + + +@requires_aiter_rowgroup +def test_sliced_page8_table_matches_rebuilt() -> None: + """A row chunk of candidates carrying their page8 table selects what the chunk with the + table rebuilt from its own rows selects.""" + torch.manual_seed(11) + case = _page8_case(2, 2048, 6, ctx_lens=[2048, 1500]) + seq_lens = case["c4_seq_lens"] + full = _rowgroup_logits(case, 6, page8=False, is_decode=True) + candidates = with_candidate_page8_table( + select_candidate_blocks_hip(full, seq_lens, topk_blocks=64, block_size=8), + seq_lens, + case["page_table"], + page_size=PAGE_SIZE, + ) + rows = slice(3, 10) + + def run(cands): + page_indices = torch.full((7, 128), -7, dtype=torch.int32, device=get_device()) + topk_within_candidate_blocks_page8( + q_fp4=case["q_fp4"][rows], + q_scale=case["q_scale"][rows], + weights=case["weights"][rows], + k_payload=case["payload8"], + k_scale=case["scale8"], + candidates=cands, + seq_lens=seq_lens[rows], + page_table=case["page_table"][rows], + page_size=PAGE_SIZE, + page_indices=page_indices, + raw_indices=None, + ) + return page_indices + + sliced = slice_candidate_blocks(candidates, rows) + assert sliced.page8 is not None + rebuilt = candidate_page8_table( + sliced.ids, + seq_lens[rows], + case["page_table"][rows], + block_size=8, + page_size=PAGE_SIZE, + ) + for got, ref in zip(sliced.page8, rebuilt): + assert torch.equal(got, ref) + # the top-k leaves its selection in no fixed order + assert _selected_slots(run(sliced)) == _selected_slots( + run(sliced._replace(page8=None)) + ) + + +def test_candidate_page8_table_moves_newest_block_last() -> None: + """The newest block trades places with the last kept id; each id maps to its 8-slot page + through the 64-slot page table; the compact length counts the full blocks plus the newest + block's reachable part; query_start_loc is one row per sequence.""" + page_table = torch.tensor( + [[5, 2, 9], [7, 0, 0], [4, 6, 0]], dtype=torch.int32, device="cuda" + ) + ids = torch.tensor( + [[3, 17, 8, 0, -1], [-1, -1, -1, -1, -1], [9, 2, 15, -1, -1]], + dtype=torch.int32, + device="cuda", + ) + seq_lens = torch.tensor([140, 0, 128], dtype=torch.int32, device="cuda") + out_ids, pages, lens, qsl = candidate_page8_table( + ids, seq_lens, page_table, block_size=8, page_size=64 + ) + assert out_ids.tolist() == [[3, 0, 8, 17, -1], [-1] * 5, [9, 2, 15, -1, -1]] + # block b lives in 64-slot page page_table[b // 8], 8-slot sub-page b % 8 + assert pages.tolist() == [ + [5 * 8 + 3, 5 * 8 + 0, 2 * 8 + 0, 9 * 8 + 1, 0], + [0] * 5, + [6 * 8 + 1, 4 * 8 + 2, 6 * 8 + 7, 0, 0], + ] + # 140 positions: block 17 covers 136..143, 4 reachable; 128: newest block 15 is full + assert lens.tolist() == [3 * 8 + 4, 0, 3 * 8] + assert qsl.tolist() == [0, 1, 2, 3] + + @pytest.mark.parametrize("is_decode", [True, False]) def test_logits_with_ragged_context_lengths(is_decode: bool) -> None: """Sizing the persistent grid for uneven contexts is the scheduler's job. @@ -1139,6 +1893,75 @@ def test_index_k_split_writer_matches_the_triton_chain(ratio: int) -> None: assert torch.equal(scale, ref_scale) +@requires_aiter_rowgroup +@pytest.mark.parametrize("writer", ["hip", "triton"]) +@pytest.mark.parametrize("ratio", [1, 2]) +def test_index_k_page8_writers_match_pack_kv_cache(writer: str, ratio: int) -> None: + """A page8 pool holds the bytes aiter's pack_kv_cache(page_size=8) lays out for the rows + the page-64 writer stores, slot for slot, and reads back as those rows.""" + from aiter.ops.flydsl.kernels.mqa_logits.pa_mqa_logits_fp4_rowgroup import ( + pack_kv_cache, + ) + + torch.manual_seed(ratio + 7 * (writer == "hip")) + num_tokens, page_size, num_pages, max_pos = 200, 64, 5, 4096 + x = (torch.randn(num_tokens, 128, device="cuda") * 3).bfloat16() + norm_weight = (1 + 0.1 * torch.randn(128, device="cuda")).bfloat16() + freqs_cis = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda") + freqs = torch.view_as_real(freqs_cis).flatten(-2) + positions = torch.randint(0, max_pos, (num_tokens,), device="cuda") + loc = torch.randperm(num_pages * page_size - 1, device="cuda")[:num_tokens] + 1 + + def write(page8: bool): + payload = torch.zeros( + num_pages, 1, 4, page_size, 16, dtype=torch.uint8, device="cuda" + ) + scale = torch.zeros( + num_pages, 1, 4, page_size, dtype=torch.uint8, device="cuda" + ) + if writer == "hip": + index_k_norm_rope_pack_store_split( + x, + norm_weight, + 1e-6, + freqs, + positions, + loc, + payload, + scale, + ratio=ratio, + page8=page8, + ) + else: + store_fp4_index_k_cache_split( + x, payload, scale, loc, page_size=page_size, rne=True, page8=page8 + ) + return payload, scale + + payload64, scale64 = write(False) + payload8, scale8 = write(True) + + slots = torch.arange(num_pages * page_size, device="cuda") + rows, packed_sf = read_fp4_index_k_split( + payload64, scale64, slots, page_size=page_size + ) + sf_bytes = torch.stack([(packed_sf >> (8 * c)) & 0xFF for c in range(4)], dim=-1) + ref_kv, ref_ks = pack_kv_cache( + rows.view(torch.uint8), sf_bytes.to(torch.uint8), page_size=8 + ) + kv8, ks8 = index_k_page8_views(payload8, scale8) + assert torch.equal(kv8.flatten(), ref_kv.flatten()) + assert torch.equal(ks8.flatten(), ref_ks.flatten()) + + rows8, packed8 = read_fp4_index_k_split( + payload8, scale8, loc, page_size=page_size, page8=True + ) + ref_rows, ref_packed = read_fp4_index_k_split( + payload64, scale64, loc, page_size=page_size + ) + assert torch.equal(rows8, ref_rows) and torch.equal(packed8, ref_packed) + + E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) TIES = [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0] RNE = [0.0, 1.0, 1.0, 2.0, 2.0, 4.0, 4.0]