From 740372bef0d35c649dbc3a38a4fb1d17d82c3d1b Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 19 Oct 2026 10:30:41 +0800 Subject: [PATCH] PR #40945: SM90 static verification indexing (grouped/length-aware indexer) Source: https://github.com/sgl-project/sglang/pull/40945 (applied via GitHub .diff, 3way) Adaptation: PR base predates #41659 which removed the Selection struct; ported sm90_decode.py and call sites to the inputs.out_page_indices interface (raw_indices=None on decode paths, reset via inputs.reset_outputs()). --- .../ops/attention/dsv4/sm90_fp4_indexer.py | 170 ++++++ .../ops/attention/dsv4/sm90_fp4_topk.py | 86 +++ .../dsv4/sm90_length_aware_indexer.py | 524 ++++++++++++++++++ python/sglang/srt/environ.py | 7 + .../layers/attention/deepseek_v4_backend.py | 27 +- .../dsv4/v41_indexer/dense_blocks.py | 22 +- .../attention/dsv4/v41_indexer/full_topk.py | 8 + .../attention/dsv4/v41_indexer/sm90_decode.py | 256 +++++++++ .../attention/dsv4/v41_indexer/types.py | 2 + .../layers/test_dsv41_candidate_blocks.py | 111 +++- 10 files changed, 1207 insertions(+), 6 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsv4/sm90_fp4_indexer.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/sm90_fp4_topk.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/sm90_length_aware_indexer.py create mode 100644 python/sglang/srt/layers/attention/dsv4/v41_indexer/sm90_decode.py diff --git a/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_indexer.py new file mode 100644 index 0000000000..b3658ce0b7 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_indexer.py @@ -0,0 +1,170 @@ +"""Fused request-to-KV mapping for opt-in SM90 32-head static verification.""" + +import torch +import triton +import triton.language as tl + +INDEX_HEAD_DIM = 128 +PAYLOAD_BYTES = tl.constexpr(64) +SCALE_BYTES = tl.constexpr(4) + + +@triton.jit +def _e2m1_decode(code): + # code: uint 0..15 -> e2m1 value. exp = bits 2..1, mantissa = bit 0, sign = bit 3. + e = (code >> 1) & 3 + m = (code & 1).to(tl.float32) + sub = m * 0.5 + exp = tl.where(e == 1, 1.0, tl.where(e == 2, 2.0, 4.0)) + nor = (1.0 + m * 0.5) * exp + v = tl.where(e == 0, sub, nor) + return tl.where((code >> 3) == 1, -v, v) + + +@triton.jit +def _fp4_index_logits_kernel( + q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd) + w_ptr, # [B, H] bf16 head weights (softmax scale folded in) + slots_ptr, # [requests, capacity * ratio] int32 request-to-token mapping + lens_ptr, # [B] int64 visible compressed positions per request + table_ptr, # [num_pages, page_size * 64 + page_size * 4] uint8 + out_ptr, # [B, L] fp32 logits, -inf beyond lens + L, + page_size, + row_stride, + stride_qb, + stride_qh, + stride_wb, + H: tl.constexpr, + HALF_D: tl.constexpr, # D // 2 == 64 nibble-pairs per row + BLOCK_L: tl.constexpr, + req_ptr, + req_stride: tl.constexpr, + ratio: tl.constexpr, +): + b = tl.program_id(0) + lb = tl.program_id(1) + offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L) + offs_h = tl.arange(0, H) + offs_i = tl.arange( + 0, HALF_D + ) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble) + + n_vis = tl.load(lens_ptr + b) + # Keep capacity-sized CUDA Graph buffers while skipping invisible tiles, + # including mapping loads, dequantization, dot products and head reduction. + if lb * BLOCK_L >= n_vis: + tl.store(out_ptr + b * L + offs_l, float("-inf"), mask=offs_l < L) + else: + valid = offs_l < tl.minimum(n_vis, L) + request = tl.load(req_ptr + b).to(tl.int64) + slot = ( + tl.load( + slots_ptr + request * req_stride + offs_l * ratio, + mask=valid, + other=0, + ).to(tl.int64) + // ratio + ) + page = slot // page_size + off = slot % page_size + row_base = page * row_stride + + # K payload: [BLOCK_L, HALF_D] uint8 + pay = tl.load( + table_ptr + + row_base[:, None] + + off[:, None] * PAYLOAD_BYTES + + offs_i[None, :], + mask=valid[:, None], + other=0, + ) + low = _e2m1_decode(pay & 0x0F) + high = _e2m1_decode((pay >> 4) & 0x0F) + # e8m0 block scales: each scale covers 16 payload bytes (32 elements). + offs_s = tl.arange(0, SCALE_BYTES) + exps = tl.load( + table_ptr + + row_base[:, None] + + page_size * PAYLOAD_BYTES + + off[:, None] * SCALE_BYTES + + offs_s[None, :], + mask=valid[:, None], + other=127, + ) + scale = tl.exp2(exps.to(tl.float32) - 127.0) + scale = scale[:, :, None] + k_shape: tl.constexpr = (BLOCK_L, SCALE_BYTES, HALF_D // SCALE_BYTES) + k_low = tl.reshape(tl.reshape(low, k_shape) * scale, (BLOCK_L, HALF_D)).to( + tl.bfloat16 + ) + k_high = tl.reshape(tl.reshape(high, k_shape) * scale, (BLOCK_L, HALF_D)).to( + tl.bfloat16 + ) + + # queries: even / odd elements, [H, HALF_D] bf16 + q_even = tl.load( + q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :] + ) + q_odd = tl.load( + q_ptr + + b * stride_qb + + offs_h[:, None] * stride_qh + + 2 * offs_i[None, :] + + 1 + ) + + acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32 + acc += tl.dot(q_odd, tl.trans(k_high)) + # reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16 + s = acc.to(tl.bfloat16).to(tl.float32) + s = tl.maximum(s, 0.0) + w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32) + s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32) + logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32) + logit = tl.where(valid, logit, float("-inf")) + tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L) + + +def fp4_index_logits_mapped_sm90( + q, + weights, + req_to_token, + req, + lens, + table, + page_size, + ratio, + width, +): + """Score with request mapping fused into the Triton kernel. + + The caller validates the common tensor/device contract. No dense slots + tensor or host read of dynamic lengths is needed during graph replay. + """ + rows, heads, _ = q.shape + out = torch.empty((rows, width), dtype=torch.float32, device=q.device) + if rows == 0 or width == 0: + return out + _fp4_index_logits_kernel[(rows, triton.cdiv(width, 64))]( + q, + weights, + req_to_token, + lens, + table, + out, + width, + page_size, + table.stride(0), + q.stride(0), + q.stride(1), + weights.stride(0), + H=heads, + HALF_D=64, + BLOCK_L=64, + req_ptr=req, + req_stride=req_to_token.stride(0), + ratio=ratio, + num_warps=4, + ) + return out diff --git a/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_topk.py b/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_topk.py new file mode 100644 index 0000000000..d6e0042e0b --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/sm90_fp4_topk.py @@ -0,0 +1,86 @@ +"""Sort selected logical positions and map slots without dense intermediates.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _sort_map_topk( + selected, + lens, + req, + mapping, + pages, + raw, + selected_row_stride: tl.constexpr, + selected_col_stride: tl.constexpr, + lens_stride: tl.constexpr, + req_stride: tl.constexpr, + mapping_row_stride: tl.constexpr, + pages_row_stride: tl.constexpr, + raw_row_stride: tl.constexpr, + K: tl.constexpr, + RATIO: tl.constexpr, + HAS_RAW: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + col = tl.arange(0, BLOCK) + logical = tl.load( + selected + row * selected_row_stride + col * selected_col_stride, + col < K, + other=0x7FFFFFFFFFFFFFFF, + ) + logical = tl.sort(logical, descending=False) + visible = tl.load(lens + row * lens_stride) + request = tl.load(req + row * req_stride).to(tl.int64) + valid = (col < K) & (logical < visible) + # Invalid candidate positions can equal the scan bound; never read their + # mappings. The backend has already filled the unused output tail with -1. + position = tl.where(valid, logical, 0) + slot = tl.load( + mapping + request * mapping_row_stride + position * RATIO, + valid, + other=0, + ) + tl.store( + pages + row * pages_row_stride + col, + tl.where(valid, slot // RATIO, -1), + col < K, + ) + if HAS_RAW: + tl.store( + raw + row * raw_row_stride + col, tl.where(valid, logical, -1), col < K + ) + + +def sort_map_topk(selected, lens, req, mapping, pages, raw, ratio): + """Publish sorted TopK values; selection and tie-breaking stay in PyTorch. + + The caller supplies int64 selected positions, with invalid candidates + replaced by the scan bound, and pre-fills both output tensors with -1. + Only the opt-in backend uses this small-K epilogue. + """ + rows, k = selected.shape + assert selected.dtype == torch.int64 and 0 < k <= 1024 + _sort_map_topk[(rows,)]( + selected, + lens, + req, + mapping, + pages, + raw if raw is not None else pages, + selected.stride(0), + selected.stride(1), + lens.stride(0), + req.stride(0), + mapping.stride(0), + pages.stride(0), + raw.stride(0) if raw is not None else 0, + K=k, + RATIO=ratio, + HAS_RAW=raw is not None, + BLOCK=triton.next_power_of_2(k), + num_warps=4, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/sm90_length_aware_indexer.py b/python/sglang/kernels/ops/attention/dsv4/sm90_length_aware_indexer.py new file mode 100644 index 0000000000..97e7c5101c --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/sm90_length_aware_indexer.py @@ -0,0 +1,524 @@ +"""Opt-in 32-head Hopper indexer with device-side prefix lengths. + +Score buffers keep their graph-stable capacity, but only the visible prefix +(and its final 64-element tile) is initialized. Every consumer MUST use the +device lengths. TopK v2 is exact by score, with arbitrary cutoff-tie selection; +it does not promise PyTorch's choice of indices with equal scores. +""" + +import torch +import triton +import triton.language as tl + +from .sm90_fp4_indexer import _e2m1_decode +from .topk import topk_transform_ragged_v2 + + +@triton.jit +def _prepare_lengths( + POS, + CAPACITY, + VISIBLE, + LENGTHS, + ROWS: tl.constexpr, + POS_STRIDE: tl.constexpr, + RATIO: tl.constexpr, + WIDTH: tl.constexpr, + HAS_CANDIDATES: tl.constexpr, +): + row = tl.program_id(0) * 256 + tl.arange(0, 256) + pos = tl.load(POS + row * POS_STRIDE, row < ROWS, other=-1) + n = tl.minimum(tl.maximum((pos + 1) // RATIO, 0), WIDTH).to(tl.int32) + tl.store(VISIBLE + row, n, row < ROWS) + if HAS_CANDIDATES: + capacity = tl.load(CAPACITY + row, row < ROWS, other=0) + tl.store(LENGTHS + row, tl.minimum(n, capacity), row < ROWS) + + +def prepare_candidate_lengths(pos, ratio, width, candidates=None): + """Fuse position conversion, clamping, casting and compact score bounds.""" + visible = torch.empty(pos.shape[0], dtype=torch.int32, device=pos.device) + lengths = torch.empty_like(visible) if candidates is not None else visible + _prepare_lengths[(triton.cdiv(pos.shape[0], 256),)]( + pos, + candidates.lengths if candidates is not None else visible, + visible, + lengths, + pos.shape[0], + pos.stride(0), + ratio, + width, + candidates is not None, + num_warps=4, + ) + return visible, lengths + + +@triton.jit +def _prefix_logits( + Q, + W, + MAPPING, + REQ, + LENS, + TABLE, + MASK, + OUT, + BLOCKS, + IS_PREFIX, + VISIBLE, + Q_STRIDE: tl.constexpr, + W_STRIDE: tl.constexpr, + MAP_STRIDE: tl.constexpr, + TABLE_STRIDE: tl.constexpr, + MASK_STRIDE: tl.constexpr, + OUT_STRIDE: tl.constexpr, + BLOCK_STRIDE: tl.constexpr, + PAGE: tl.constexpr, + RATIO: tl.constexpr, + HAS_MASK: tl.constexpr, + HAS_BLOCKS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + n = tl.load(LENS + row) + request = tl.load(REQ + row).to(tl.int64) + h = tl.arange(0, 32) + d = tl.arange(0, 64) + # Reuse query/weights across the tiles owned by this CTA. + qe = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :]) + qo = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :] + 1) + weight = tl.load(W + row * W_STRIDE + h).to(tl.float32) + if HAS_BLOCKS: + visible_n = tl.load(VISIBLE + row) + is_prefix = tl.load(IS_PREFIX + row) + for tile in range(tl.program_id(1), tl.cdiv(n, 64), tl.num_programs(1)): + col = tile * 64 + tl.arange(0, 64) + visible = col < n + valid = visible + position = col + if HAS_BLOCKS: + # If all selected blocks form a prefix, keep the short-context + # scoring path's contiguous accesses and avoid the block gather. + if is_prefix == 0: + block = tl.load( + BLOCKS + row * BLOCK_STRIDE + col // BLOCK_SIZE, + visible, + other=-1, + ) + position = block * BLOCK_SIZE + col % BLOCK_SIZE + valid = valid & (position >= 0) & (position < visible_n) + if HAS_MASK: + valid = valid & tl.load( + MASK + row * MASK_STRIDE + col, visible, other=False + ) + if tl.sum(valid.to(tl.int32), 0) > 0: + slot = ( + tl.load( + MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0 + ).to(tl.int64) + // RATIO + ) + base = (slot // PAGE) * TABLE_STRIDE + off = slot % PAGE + payload = tl.load( + TABLE + base[:, None] + off[:, None] * 64 + d[None, :], + valid[:, None], + other=0, + ) + scales = tl.load( + TABLE + + base[:, None] + + PAGE * 64 + + off[:, None] * 4 + + tl.arange(0, 4)[None, :], + valid[:, None], + other=127, + ) + scale = tl.exp2(scales.to(tl.float32) - 127.0)[:, :, None] + lo = _e2m1_decode(payload & 15) + hi = _e2m1_decode(payload >> 4) + kl = tl.reshape(tl.reshape(lo, (64, 4, 16)) * scale, (64, 64)).to( + tl.bfloat16 + ) + kh = tl.reshape(tl.reshape(hi, (64, 4, 16)) * scale, (64, 64)).to( + tl.bfloat16 + ) + acc = tl.dot(qe, tl.trans(kl)) + acc += tl.dot(qo, tl.trans(kh)) + score = tl.maximum(acc.to(tl.bfloat16).to(tl.float32), 0.0) + score = (score * weight[:, None]).to(tl.bfloat16).to(tl.float32) + score = tl.sum(score, 0).to(tl.bfloat16).to(tl.float32) + score = tl.where(valid, score, -float("inf")) + else: + score = tl.full((64,), -float("inf"), tl.float32) + # TopK uses 16-byte loads at the prefix boundary. Initialize the whole + # last tile, including the <= 3 padding floats beyond the logical width. + tl.store(OUT + row * OUT_STRIDE + col, score, col < OUT_STRIDE) + + +def prefix_logits( + q, + weights, + mapping, + req, + lens, + table, + page, + ratio, + width, + mask=None, + *, + candidates=None, + visible=None, +): + """Return partially initialized scores, bounded by int32 device lengths. + + lens bounds the score buffer in [0, width]. With candidates, visible is the + original prefix length and lens bounds the compact candidate positions. + """ + assert candidates is None or (visible is not None and mask is None) + rows = q.shape[0] + stride = triton.cdiv(width, 4) * 4 + out = torch.empty((rows, stride), dtype=torch.float32, device=q.device) + sm_count = torch.cuda.get_device_properties(q.device).multi_processor_count + tiles = min( + triton.cdiv(width, 64), + max(8, triton.next_power_of_2(triton.cdiv(8 * sm_count, rows))), + ) + _prefix_logits[(rows, tiles)]( + q, + weights, + mapping, + req, + lens, + table, + mask if mask is not None else out, + out, + candidates.blocks if candidates is not None else out, + candidates.is_prefix if candidates is not None else lens, + visible if visible is not None else lens, + q.stride(0), + weights.stride(0), + mapping.stride(0), + table.stride(0), + mask.stride(0) if mask is not None else 0, + stride, + candidates.blocks.stride(0) if candidates is not None else 0, + page, + ratio, + mask is not None, + candidates is not None, + candidates.block_size if candidates is not None else 1, + num_warps=4, + ) + return out + + +def select_prefix_topk(scores, lens, k): + """Exact TopK by score over each visible prefix; -1 pads short rows.""" + selected = torch.empty( + (scores.shape[0], k), dtype=torch.int32, device=scores.device + ) + # The paged small-batch dispatch can request a 16-CTA cluster, while H20 + # supports at most 8. Reuse v2's length-aware non-cluster entry point. + offsets = torch.zeros_like(lens) + topk_transform_ragged_v2(scores, lens, out_offsets=offsets, out_indices=selected) + return selected + + +@triton.jit +def _block_scores( + SCORES, + LENS, + OUT, + BLOCK_LENS, + SCORE_STRIDE: tl.constexpr, + OUT_STRIDE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + TILE: tl.constexpr, +): + row = tl.program_id(0) + n = tl.load(LENS + row) + count = tl.cdiv(n, BLOCK_SIZE) + if tl.program_id(1) == 0: + tl.store(BLOCK_LENS + row, count) + for tile in range(tl.program_id(1), tl.cdiv(count, TILE), tl.num_programs(1)): + block = tile * TILE + tl.arange(0, TILE) + pos = block[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] + v = tl.load(SCORES + row * SCORE_STRIDE + pos, pos < n, other=-float("inf")) + score = tl.max(v, 1) + score = tl.where(block == count - 1, float("inf"), score) + tl.store(OUT + row * OUT_STRIDE + block, score, block < OUT_STRIDE) + + +@triton.jit +def _publish_blocks( + SELECTED, + SCORES, + MASK, + K: tl.constexpr, + SCORE_STRIDE: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + i = tl.program_id(1) * 256 + tl.arange(0, 256) + block = tl.load(SELECTED + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1) + valid = (i < K * BLOCK_SIZE) & (block >= 0) + score = tl.load( + SCORES + row * SCORE_STRIDE + tl.maximum(block, 0), valid, other=-float("inf") + ) + pos = block * BLOCK_SIZE + i % BLOCK_SIZE + # Keep the whole last block, exactly as the existing candidate mask does; + # visibility is enforced independently by every score consumer. + tl.store( + MASK + row * WIDTH + pos, True, valid & (score > -float("inf")) & (pos < WIDTH) + ) + + +def _candidate_block_topk(scores, lens, width, topk_blocks, block_size): + rows = scores.shape[0] + blocks = triton.cdiv(width, block_size) + stride = triton.cdiv(blocks, 4) * 4 + block_scores = torch.empty( + (rows, stride), dtype=torch.float32, device=scores.device + ) + block_lens = torch.empty_like(lens) + _block_scores[(rows, min(16, triton.cdiv(blocks, 128)))]( + scores, + lens, + block_scores, + block_lens, + scores.stride(0), + stride, + block_size, + 128, + num_warps=4, + ) + k = min(topk_blocks, blocks) + selected = select_prefix_topk(block_scores, block_lens, k) + return selected, block_scores + + +def candidate_mask(scores, lens, width, topk_blocks, block_size): + selected, block_scores = _candidate_block_topk( + scores, lens, width, topk_blocks, block_size + ) + rows, k = selected.shape + # Preserve CandidateMasks' full-width contract for fallback consumers. Only + # the source layer initializes this bool buffer; consumers read valid tiles. + mask = torch.zeros((rows, width), dtype=torch.bool, device=scores.device) + _publish_blocks[(rows, triton.cdiv(k * block_size, 256))]( + selected, + block_scores, + mask, + k, + block_scores.stride(0), + width, + block_size, + num_warps=4, + ) + return mask + + +@triton.jit +def _sort_blocks( + SELECTED, + SCORES, + LENS, + BLOCKS, + LENGTHS, + IS_PREFIX, + K: tl.constexpr, + SCORE_STRIDE: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + i = tl.arange(0, BLOCK) + count = tl.cdiv(tl.load(LENS + row), BLOCK_SIZE) + if count <= K: + # Short contexts keep every finite block. Avoid a 2048-element bitonic + # sort when the candidate list is already the contiguous prefix. + block = i + valid = (i < K) & (i < count) + else: + block = tl.load(SELECTED + row * K + i, i < K, other=-1) + valid = (i < K) & (block >= 0) + score = tl.load( + SCORES + row * SCORE_STRIDE + tl.maximum(block, 0), + valid, + other=-float("inf"), + ) + valid = valid & (score > -float("inf")) + block = tl.where(valid, block, 0x7FFFFFFF) + # All -inf blocks are excluded even when K covers the prefix. Such holes + # need compaction just like the selected subset of a long context does. + if count > K or tl.sum(valid.to(tl.int32), 0) != count: + block = tl.sort(block, descending=False) + valid = block != 0x7FFFFFFF + count = tl.sum(valid.to(tl.int32), 0) + tl.store(BLOCKS + row * K + i, tl.where(valid, block, -1), i < K) + # Keep entire selected blocks, matching CandidateMasks. Consumer visibility + # independently masks the final partial block and any differing ratio. + tl.store(LENGTHS + row, tl.minimum(count * BLOCK_SIZE, WIDTH)) + tl.store(IS_PREFIX + row, tl.sum((valid & (block != i)).to(tl.int32), 0) == 0) + + +def candidate_blocks(scores, lens, width, topk_blocks, block_size): + """Publish a compact list using prefix scores and non-cluster TopK.""" + from sglang.srt.layers.attention.dsv4.v41_indexer.sm90_decode import CandidateBlocks + + selected, block_scores = _candidate_block_topk( + scores, lens, width, topk_blocks, block_size + ) + rows, k = selected.shape + blocks = torch.empty_like(selected) + lengths = torch.empty_like(lens) + is_prefix = torch.empty_like(lens) + _sort_blocks[(rows,)]( + selected, + block_scores, + lens, + blocks, + lengths, + is_prefix, + k, + block_scores.stride(0), + width, + block_size, + triton.next_power_of_2(k), + num_warps=4, + ) + return CandidateBlocks(blocks, lengths, is_prefix, width, block_size) + + +@triton.jit +def _blocks_to_mask( + BLOCKS, + MASK, + K: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + i = tl.program_id(1) * 256 + tl.arange(0, 256) + block = tl.load(BLOCKS + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1) + pos = block * BLOCK_SIZE + i % BLOCK_SIZE + tl.store( + MASK + row * WIDTH + pos, + True, + (i < K * BLOCK_SIZE) & (block >= 0) & (pos < WIDTH), + ) + + +def materialize_candidate_mask(candidates): + """Build the legacy mask only when a later layer takes a fallback path.""" + rows, k = candidates.blocks.shape + mask = torch.zeros( + (rows, candidates.width), dtype=torch.bool, device=candidates.blocks.device + ) + _blocks_to_mask[(rows, triton.cdiv(k * candidates.block_size, 256))]( + candidates.blocks, + mask, + k, + candidates.width, + candidates.block_size, + num_warps=4, + ) + return mask + + +@triton.jit +def _publish_topk( + SELECTED, + SCORES, + LENS, + REQ, + MAPPING, + PAGES, + RAW, + BLOCKS, + K: tl.constexpr, + SCORE_STRIDE: tl.constexpr, + MAP_STRIDE: tl.constexpr, + PAGE_STRIDE: tl.constexpr, + RAW_STRIDE: tl.constexpr, + BLOCK_STRIDE: tl.constexpr, + RATIO: tl.constexpr, + HAS_RAW: tl.constexpr, + FILTER_MASKED: tl.constexpr, + HAS_BLOCKS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + col = tl.arange(0, BLOCK) + logical = tl.load(SELECTED + row * K + col, col < K, other=-1) + n = tl.load(LENS + row) + valid = (logical >= 0) & (logical < n) & (col < K) + if FILTER_MASKED: + score = tl.load( + SCORES + row * SCORE_STRIDE + tl.maximum(logical, 0), + valid, + other=-float("inf"), + ) + valid = valid & (score > -float("inf")) + if HAS_BLOCKS: + block = tl.load( + BLOCKS + row * BLOCK_STRIDE + tl.maximum(logical, 0) // BLOCK_SIZE, + valid, + other=-1, + ) + logical = block * BLOCK_SIZE + logical % BLOCK_SIZE + logical = tl.sort(tl.where(valid, logical, 0x7FFFFFFF), descending=False) + valid = (logical != 0x7FFFFFFF) & (col < K) + request = tl.load(REQ + row).to(tl.int64) + position = tl.where(valid, logical, 0).to(tl.int64) + slot = tl.load(MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0) + tl.store( + PAGES + row * PAGE_STRIDE + col, tl.where(valid, slot // RATIO, -1), col < K + ) + if HAS_RAW: + tl.store(RAW + row * RAW_STRIDE + col, tl.where(valid, logical, -1), col < K) + + +def publish_topk( + selected, + scores, + lens, + req, + mapping, + pages, + raw, + ratio, + filter_masked, + *, + candidates=None, +): + rows, k = selected.shape + _publish_topk[(rows,)]( + selected, + scores, + lens, + req, + mapping, + pages, + raw if raw is not None else pages, + candidates.blocks if candidates is not None else selected, + k, + scores.stride(0), + mapping.stride(0), + pages.stride(0), + raw.stride(0) if raw is not None else 0, + candidates.blocks.stride(0) if candidates is not None else 0, + ratio, + raw is not None, + filter_masked, + candidates is not None, + candidates.block_size if candidates is not None else 1, + triton.next_power_of_2(k), + num_warps=4, + ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 51bbaac3fa..cb8de48ff7 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1579,6 +1579,13 @@ class Envs: # Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead # of the DeepGEMM dense fp4 logits kernel (test oracle / fallback). SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False) + # Fused mapping for SM90 32-head, ratio-1/2 static target verification. + SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER = EnvBool(False) + # Persistent prefix scoring and length-aware TopK; requires GROUPED_INDEXER. + # Equal-score cutoff ties can select different positions than PyTorch. + SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False) + # Compact candidate lists; requires the length-aware static-verify path. + SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 204434813b..99dda6a6e6 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -3131,7 +3131,7 @@ class DeepseekV4AttnBackend( is_source = is_consumer = False inputs = self._make_low_ratio_decode_indexer_inputs( - layer, x, q_lora, req, pos, mode + layer, x, q_lora, req, pos, forward_batch ) if is_source: published = self.decode_candidates.publish_decode(inputs) @@ -3179,7 +3179,9 @@ class DeepseekV4AttnBackend( ) self.full_topk_indexer.topk_prefill_captured(inputs) - def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode): + def _make_low_ratio_decode_indexer_inputs( + self, layer, x, q_lora, req, pos, forward_batch + ): ratio = layer.compress_ratio metadata = ( self.forward_metadata.c1_indexer_metadata @@ -3187,6 +3189,24 @@ class DeepseekV4AttnBackend( else self.forward_metadata.c2_indexer_metadata ) assert metadata is not None, f"no decode indexer metadata for {ratio = }" + group_size = 1 + if ( + envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get() + and forward_batch.forward_mode.is_target_verify() + and not self.is_dspark_draft + and read_ragged_verify_mode() is RaggedVerifyMode.STATIC + and self.forward_metadata.late_layer_tail is None + and self.speculative_num_draft_tokens is not None + and self.speculative_num_draft_tokens > 1 + and forward_batch.spec_info.draft_token_num + == self.speculative_num_draft_tokens + and req.shape[0] + == forward_batch.batch_size * self.speculative_num_draft_tokens + and x.shape[0] == q_lora.shape[0] == req.shape[0] + ): + # Only static verify guarantees fixed request-major groups. + # Do not read GPU request ids or lengths during graph capture. + group_size = self.speculative_num_draft_tokens return DecodeInputs( indexer=layer.indexer, layer_id=layer.layer_id, @@ -3197,7 +3217,8 @@ class DeepseekV4AttnBackend( positions=pos, req_rows=req, paged_metadata=metadata, - is_verify=mode.is_target_verify(), + is_verify=forward_batch.forward_mode.is_target_verify(), + group_size=group_size, out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices( ratio ), diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py index 3b37b5aae5..db1883b3c5 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py @@ -3,7 +3,7 @@ its best blocks, a consumer takes its top-k among them.""" from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple, Union import msgspec import torch @@ -27,6 +27,7 @@ from .scoring import ( write_decode, write_prefill, ) +from .sm90_decode import CandidateBlocks, try_sm90_decode from .types import ( CandidateMetadata, DecodeInputs, @@ -93,6 +94,14 @@ class DenseBlocksBackend: self._torch_consume_prefill(inputs, published) def publish_decode(self, inputs: DecodeInputs): + handled, published = try_sm90_decode( + inputs=inputs, + token_to_kv_pool=self.token_to_kv_pool, + req_to_token=self.req_to_token, + is_source=True, + ) + if handled: + return published d = decode_scores( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, @@ -128,8 +137,17 @@ class DenseBlocksBackend: def consume_decode( self, inputs: DecodeInputs, - published: Optional[BlockIds], + published: Optional[Union[BlockIds, CandidateBlocks]], ) -> None: + handled, _ = try_sm90_decode( + inputs=inputs, + token_to_kv_pool=self.token_to_kv_pool, + req_to_token=self.req_to_token, + is_consumer=True, + published=published, + ) + if handled: + return d = decode_scores( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py index fbf8e3acdd..3acb7e08bb 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py @@ -28,6 +28,7 @@ from .scoring import ( select_decode, write_prefill, ) +from .sm90_decode import try_sm90_decode from .types import ( CapturedPrefillInputs, DecodeInputs, @@ -134,6 +135,13 @@ class FullTopKIndexer: ) def _torch_decode(self, inputs: DecodeInputs) -> None: + handled, _ = try_sm90_decode( + inputs=inputs, + token_to_kv_pool=self.token_to_kv_pool, + req_to_token=self.req_to_token, + ) + if handled: + return d = decode_scores( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sm90_decode.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sm90_decode.py new file mode 100644 index 0000000000..9600190040 --- /dev/null +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sm90_decode.py @@ -0,0 +1,256 @@ +"""Opt-in SM90 static-verification selection for the V4.1 decode backends.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Tuple + +import torch + +from sglang.srt.environ import envs + +from .types import CandidateMetadata, DecodeInputs + +if TYPE_CHECKING: + from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool + + +@dataclass +class CandidateBlocks(CandidateMetadata): + """Sorted block IDs with device lengths refreshed on each graph replay. + + Ordinary decode consumers can use ``blocks`` directly. Length-aware + consumers also use the device lengths; never cache a derived full mask. + """ + + blocks: torch.Tensor # [rows, topk_blocks] int32, -1 padding + lengths: torch.Tensor # [rows] int32, selected capacity in positions + is_prefix: torch.Tensor # [rows] int32, blocks are consecutive from zero + width: int # source capacity in compressed positions + block_size: int + + +def _candidate_mask(published, width: int, block_size: int) -> torch.Tensor: + """Adapt a standard block-ID publish to prefix scoring, without caching.""" + if isinstance(published, CandidateBlocks): + from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import ( + materialize_candidate_mask, + ) + + assert published.width >= width and published.block_size == block_size + return materialize_candidate_mask(published) + blocks = published.blocks + num_blocks = (width + block_size - 1) // block_size + keep = torch.zeros( + (*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device + ) + ids = blocks.to(torch.int64) + ids = ids.masked_fill((ids < 0) | (ids >= num_blocks), num_blocks) + keep.scatter_(-1, ids, True) + return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width] + + +def try_sm90_decode( + *, + inputs: DecodeInputs, + token_to_kv_pool: DeepSeekV4TokenToKVPool, + req_to_token: torch.Tensor, + is_source: bool = False, + is_consumer: bool = False, + published: Optional[CandidateMetadata] = None, +) -> Tuple[bool, Optional[CandidateMetadata]]: + """Return whether selection was handled, plus a source's published blocks. + + Candidate roles come from the caller: graph variants may bypass filtering + even for a layer whose indexer normally publishes or consumes candidates. + """ + ratio = inputs.compress_ratio + if not ( + envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get() + and inputs.is_verify + and inputs.group_size > 1 + and ratio in (1, 2) + ): + return False, None + indexer = inputs.indexer + req, pos = inputs.req_rows, inputs.positions + bs = req.shape[0] + assert pos.shape[0] == bs, ( + f"decode expects one token per request, {pos.shape=} {bs=}" + ) + # Metadata reserves graph replay capacity; visibility remains on device. + lmax = min( + inputs.paged_metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio + ) + if bs == 0 or lmax == 0: + inputs.reset_outputs() + return True, None + q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos]) + weights = indexer.head_weights(inputs.x) + table = token_to_kv_pool.get_index_k_with_scale_buffer(inputs.layer_id) + if not ( + q.is_cuda + and torch.version.cuda is not None + and torch.cuda.get_device_capability(q.device)[0] == 9 + and q.dtype == weights.dtype == torch.bfloat16 + and q.shape[1:] == (32, 128) + and q.is_contiguous() + and weights.is_contiguous() + and req_to_token.dtype == torch.int32 + and req_to_token.stride(1) == 1 + and table.dtype == torch.uint8 + and table.dim() == 2 + and table.stride(1) == 1 + ): + return False, None + + from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( + select_candidate_block_ids, + topk_among_blocks, + ) + + # Imported only after dispatch to avoid a module-initialization cycle. + from .dense_blocks import BlockIds + + inputs.reset_outputs() + if is_consumer: + assert published is not None and published.blocks.shape[0] == bs + use_length_aware = ( + envs.SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER.get() + and 0 < indexer.index_topk <= 2048 + and ( + not is_source + or ( + 0 < indexer.candidate_topk_blocks <= 2048 + and indexer.candidate_block_size in (1, 2, 4, 8, 16, 32, 64, 128) + ) + ) + ) + if use_length_aware: + from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import ( + candidate_blocks, + prefix_logits, + prepare_candidate_lengths, + publish_topk, + select_prefix_topk, + ) + + request = req.to(torch.int64).contiguous() + compact = envs.SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES.get() and ( + is_source or is_consumer + ) + candidates = ( + published + if compact and is_consumer and isinstance(published, CandidateBlocks) + else None + ) + if compact: + visible, score_lens = prepare_candidate_lengths( + pos, ratio, lmax, candidates + ) + else: + visible = ((pos + 1) // ratio).clamp(0, lmax).to(torch.int32).contiguous() + score_lens = visible + consume = None + if is_consumer and candidates is None: + consume = _candidate_mask(published, lmax, indexer.candidate_block_size) + assert consume.shape[0] == bs and consume.shape[1] >= lmax + assert consume.stride(1) == 1 + score_width = lmax + if candidates is not None: + assert candidates.width >= lmax + assert candidates.block_size == indexer.candidate_block_size + score_width = min(lmax, candidates.blocks.shape[1] * candidates.block_size) + scores = prefix_logits( + q, + weights, + req_to_token, + request, + score_lens, + table, + table.shape[1] // 68, + ratio, + score_width, + consume, + candidates=candidates, + visible=visible, + ) + result = None + if is_source: + result = candidate_blocks( + scores, + visible, + lmax, + indexer.candidate_topk_blocks, + indexer.candidate_block_size, + ) + if not compact: + # Main's decode protocol carries block IDs even without the + # compact scoring flag. Drop only the optional device lengths. + result = BlockIds(blocks=result.blocks) + idx = select_prefix_topk( + scores, score_lens, min(indexer.index_topk, score_width) + ) + publish_topk( + idx, + scores, + score_lens, + request, + req_to_token, + inputs.out_page_indices, + None, + ratio, + is_consumer, + candidates=candidates, + ) + return True, result + + from sglang.kernels.ops.attention.dsv4.sm90_fp4_indexer import ( + fp4_index_logits_mapped_sm90, + ) + + lens = (pos + 1) // ratio + scores = fp4_index_logits_mapped_sm90( + q, + weights, + req_to_token, + req.to(torch.int64).contiguous(), + lens.to(torch.int64).contiguous(), + table, + table.shape[1] // 68, + ratio, + lmax, + ) + result = None + if is_source: + result = BlockIds( + blocks=select_candidate_block_ids( + scores, + lens[:, None], + indexer.candidate_topk_blocks, + indexer.candidate_block_size, + ) + ) + k = min(indexer.index_topk, lmax) + if is_consumer: + idx = topk_among_blocks( + scores, lens, published.blocks, k, block_size=indexer.candidate_block_size + ) + idx = idx.masked_fill(idx < 0, lmax) + else: + idx = scores.topk(k, dim=-1, sorted=False).indices + if 0 < k <= 1024: + from sglang.kernels.ops.attention.dsv4.sm90_fp4_topk import sort_map_topk + + sort_map_topk( + idx, lens, req, req_to_token, inputs.out_page_indices, None, ratio + ) + else: + idx = idx.sort(dim=-1).values + reach = idx < lens[:, None] + slots = ( + req_to_token[req[:, None], idx.clamp_max(lmax - 1) * ratio].to(torch.int64) + // ratio + ) + inputs.out_page_indices[:bs, :k] = torch.where(reach, slots, -1).to(torch.int32) + return True, result diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py index 7800b366ac..f02795e0b3 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py @@ -81,6 +81,8 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True): req_rows: torch.Tensor # [rows] req_to_token row of each query row paged_metadata: PagedIndexerMetadata is_verify: bool + # Fixed request-major groups, validated by the attention backend for static verify. + group_size: int = 1 # [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded. out_page_indices: torch.Tensor diff --git a/test/registered/unit/layers/test_dsv41_candidate_blocks.py b/test/registered/unit/layers/test_dsv41_candidate_blocks.py index 39d023876f..f366efe459 100644 --- a/test/registered/unit/layers/test_dsv41_candidate_blocks.py +++ b/test/registered/unit/layers/test_dsv41_candidate_blocks.py @@ -1,4 +1,6 @@ import unittest +from types import SimpleNamespace +from unittest.mock import patch import torch @@ -6,8 +8,19 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( select_candidate_block_ids, topk_among_blocks, ) +from sglang.srt.environ import envs +from sglang.srt.layers.attention.dsv4.v41_indexer import dense_blocks from sglang.srt.layers.attention.dsv4.v41_indexer.dense_blocks import BlockIds -from sglang.srt.layers.attention.dsv4.v41_indexer.types import get_tail_row_indices +from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import DecodeScores +from sglang.srt.layers.attention.dsv4.v41_indexer.sm90_decode import ( + CandidateBlocks, + _candidate_mask, + try_sm90_decode, +) +from sglang.srt.layers.attention.dsv4.v41_indexer.types import ( + Selection, + get_tail_row_indices, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -116,5 +129,101 @@ class TestBlockIdsTail(CustomTestCase): ) +class TestSm90CandidateCompatibility(CustomTestCase): + def test_standard_blocks_to_prefix_mask_tracks_updates(self): + blocks = torch.tensor([[2, 0, -1], [-1, -1, -1]], dtype=torch.int32) + published = BlockIds(blocks=blocks) + mask = _candidate_mask(published, width=11, block_size=4) + expected = torch.tensor([[True] * 4 + [False] * 4 + [True] * 3, [False] * 11]) + torch.testing.assert_close(mask, expected) + # A later replay updates the same block storage; conversion must read it. + blocks[0] = torch.tensor([1, -1, -1]) + expected[0] = torch.tensor([False] * 4 + [True] * 4 + [False] * 3) + torch.testing.assert_close( + _candidate_mask(published, width=11, block_size=4), expected + ) + + def test_compact_publish_is_consumed_by_ordinary_decode(self): + scores = torch.tensor( + [[1.0, 4.0, 90.0, 80.0, 3.0, 2.0], [60.0, 50.0, 7.0, 99.0, 80.0, 70.0]] + ) + slots = torch.tensor([[10, 11, 12, 13, 14, 15], [20, 21, 22, 23, 24, 25]]) + d = DecodeScores( + bs=2, lmax=6, lens=torch.tensor([6, 3]), slots=slots, scores=scores + ) + published = CandidateBlocks( + blocks=torch.tensor([[0, 2], [1, -1]], dtype=torch.int32), + lengths=torch.tensor([4, 2], dtype=torch.int32), + is_prefix=torch.tensor([0, 0], dtype=torch.int32), + width=6, + block_size=2, + ) + backend = dense_blocks.DenseBlocksBackend( + token_to_kv_pool=None, + req_to_token=None, + candidate_topk_blocks=2, + candidate_block_size=2, + use_deep_gemm_prefill=False, + ) + inputs = SimpleNamespace( + indexer=SimpleNamespace(index_topk=3), + compress_ratio=1, + is_verify=False, + group_size=1, + ) + for with_raw in (True, False): + with self.subTest(with_raw=with_raw): + out = Selection( + page_indices=torch.full((2, 3), -1, dtype=torch.int32), + raw_indices=( + torch.full((2, 3), -1, dtype=torch.int32) if with_raw else None + ), + ) + with ( + envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.override(True), + patch.object(dense_blocks, "decode_scores", return_value=d), + ): + backend.consume_decode(inputs, published, out) + torch.testing.assert_close( + out.page_indices, + torch.tensor([[11, 14, 15], [22, -1, -1]], dtype=torch.int32), + ) + if with_raw: + torch.testing.assert_close( + out.raw_indices, + torch.tensor([[1, 4, 5], [2, -1, -1]], dtype=torch.int32), + ) + + def test_unsupported_batches_preserve_fallback_outputs(self): + for enabled, verify, group_size, ratio in ( + (False, True, 5, 1), + (True, False, 5, 1), + (True, True, 1, 1), + (True, True, 5, 4), + ): + with self.subTest( + enabled=enabled, verify=verify, group_size=group_size, ratio=ratio + ): + out = Selection( + page_indices=torch.full((2, 3), 42, dtype=torch.int32), + raw_indices=torch.full((2, 3), 24, dtype=torch.int32), + ) + with envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.override(enabled): + handled, published = try_sm90_decode( + inputs=SimpleNamespace( + compress_ratio=ratio, + is_verify=verify, + group_size=group_size, + ), + out=out, + token_to_kv_pool=None, + req_to_token=None, + ) + self.assertFalse(handled) + self.assertIsNone(published) + self.assertTrue((out.page_indices == 42).all()) + self.assertTrue((out.raw_indices == 24).all()) + + if __name__ == "__main__": unittest.main()