diff --git a/python/sglang/kernels/ops/attention/dsv4/index_logits.py b/python/sglang/kernels/ops/attention/dsv4/index_logits.py index b24711a14b..0a0f7db166 100644 --- a/python/sglang/kernels/ops/attention/dsv4/index_logits.py +++ b/python/sglang/kernels/ops/attention/dsv4/index_logits.py @@ -4,6 +4,7 @@ memory budget), the paged pool, or the published blocks of a sparse table.""" from __future__ import annotations +import functools from typing import Iterator, Tuple import torch @@ -17,10 +18,34 @@ from sglang.srt.utils.common import ceil_align from .candidate_table import CANDIDATE_BLOCK_SIZE -_LOGITS_DTYPE = ( - torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32 -) -_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize + +@functools.cache +def _is_sm100() -> bool: + return ( + torch.cuda.is_available() + and torch.version.cuda is not None + and torch.cuda.get_device_capability()[0] == 10 + ) + + +@functools.cache +def index_logits_dtype() -> torch.dtype: + """bf16 on SM100, where the reference kernels score in bf16; fp32 elsewhere. + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS, when set, overrides the choice.""" + forced = envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() + bf16 = _is_sm100() if forced is None else forced + return torch.bfloat16 if bf16 else torch.float32 + + +@functools.cache +def index_weights_dtype() -> torch.dtype: + """DeepGEMM reduces in the head weights' dtype and takes bf16 weights only on + SM100 with bf16 logits.""" + if index_logits_dtype() == torch.bfloat16 and _is_sm100(): + return torch.bfloat16 + return torch.float32 + + # Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which # DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row) FLAT_LOGITS_WIDTH_ALIGN = 64 @@ -32,16 +57,15 @@ def flat_index_logits_rows_per_tile( *, heads: int, budget_bytes: int, - item_size: int = _LOGITS_DTYPE_BYTES, ) -> int: - """Rows per logits tile within ``budget_bytes`` (sized as fp32, which also - bounds bf16), at the kernel's row alignment.""" + """Rows per logits tile within ``budget_bytes``, at the kernel's row alignment. + Tiles are sized as fp32 logits whatever the logits dtype: bf16 tiles then fill + half the budget, which leaves room for the per-row selection scratch that + grows with the tile.""" row_alignment = 128 // heads - # NOTE: for bf16, the estimated row bytes is 2 times the actual bytes - effective_width = width * item_size // 4 rows_per_chunk = mqa_logits_rows_per_chunk( num_rows=ceil_align(rows, row_alignment), - row_bytes=mqa_logits_row_bytes(effective_width), + row_bytes=mqa_logits_row_bytes(width), budget_bytes=budget_bytes, ) if rows_per_chunk is None: @@ -72,6 +96,8 @@ def flat_index_logits_tiles( rows_per_chunk = flat_index_logits_rows_per_tile( rows, width, heads=q[0].shape[1], budget_bytes=budget_bytes ) + weights = weights.to(index_weights_dtype()) + logits_dtype = index_logits_dtype() for offset in range(0, rows, rows_per_chunk): tile = slice(offset, min(offset + rows_per_chunk, rows)) tile_starts = starts[tile] @@ -85,7 +111,7 @@ def flat_index_logits_tiles( tile_starts + lengths[tile], False, width, - logits_dtype=_LOGITS_DTYPE, + logits_dtype=logits_dtype, ), ) @@ -108,13 +134,13 @@ def deep_gemm_fp4_paged_mqa_logits( return fp8_fp4_paged_mqa_logits( q_fp4, k_cache, - weights, + weights.to(index_weights_dtype()), sl, page_table, deep_gemm_metadata, max_seq_len, False, - logits_dtype=_LOGITS_DTYPE, + logits_dtype=index_logits_dtype(), ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d297fd0a46..10f20eb799 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1748,7 +1748,8 @@ class Envs: SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32") # DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead # of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16. - SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False) + # Unset: bf16 on SM100 (with bf16 head weights), fp32 elsewhere. + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(None) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) # gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16 diff --git a/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py b/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py index 3f74f4e9e1..db11598ad5 100644 --- a/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py +++ b/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py @@ -111,16 +111,23 @@ def request_blocks(inputs, candidates): def dense_scores(inputs): from deep_gemm import fp8_fp4_mqa_logits + from sglang.kernels.ops.attention.dsv4.index_logits import ( + index_logits_dtype, + index_weights_dtype, + ) + width = (max(n for _, n in inputs["request_lengths"]) + 3) // 4 * 4 + # Score in the dtypes the indexer uses on this GPU, so the reference matches. scores = fp8_fp4_mqa_logits( inputs["q"], inputs["kv"], - inputs["weights"], + inputs["weights"].to(index_weights_dtype()), inputs["starts"], inputs["starts"] + inputs["lengths"], False, width, - ) + logits_dtype=index_logits_dtype(), + ).float() return scores.masked_fill_( torch.arange(width, device="cuda")[None, :] >= inputs["lengths"][:, None], -torch.inf, diff --git a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py index ce117e621d..43bb90bd2b 100644 --- a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py @@ -49,6 +49,11 @@ def make_case(rows, ctx, seed) -> Case: index K in the pool page layout [pages, PAGE * 64 payload | PAGE * 4 scale].""" from deep_gemm import fp8_fp4_mqa_logits + from sglang.kernels.ops.attention.dsv4.index_logits import ( + index_logits_dtype, + index_weights_dtype, + ) + torch.manual_seed(seed) dev = "cuda" n_slots = (ctx + PAGE - 1) // PAGE * PAGE @@ -72,15 +77,17 @@ def make_case(rows, ctx, seed) -> Case: lens[0] = 0 # a query that sees no compressed position yet width = (ctx + 7) // 8 * 8 starts = torch.zeros(rows, device=dev, dtype=torch.int32) + # Score in the dtypes the indexer uses on this GPU, so the reference matches. dense = fp8_fp4_mqa_logits( (q_fp4, q_sf), (k_fp4[:width], k_sf[:width]), - weights, + weights.to(index_weights_dtype()), starts, lens, False, width, - ) + logits_dtype=index_logits_dtype(), + ).float() data = DeepGEMMPrefillData( k_slots=torch.arange(width, device=dev), request_starts=starts, @@ -283,6 +290,20 @@ def logits_of(table, data: DeepGEMMPrefillData, k_cache: torch.Tensor): ) +def block_scores(case: Case) -> torch.Tensor: + """[rows, blocks] max dense score per block, the score blocks are ranked by.""" + j = torch.arange(case.dense.shape[1], device=case.dense.device) + scores = case.dense.masked_fill(j[None, :] >= case.lens[:, None], -torch.inf) + pad = -scores.shape[1] % BLOCK + if pad: + scores = torch.nn.functional.pad(scores, (0, pad), value=-torch.inf) + scores = scores.unflatten(-1, (-1, BLOCK)).amax(dim=-1) + # the block of the last causal position is always kept + last = (case.lens.long() - 1) // BLOCK + blocks = torch.arange(scores.shape[1], device=scores.device) + return scores.masked_fill(blocks[None, :] == last[:, None], torch.inf) + + def picks(positions: torch.Tensor, row: int) -> set: return set(positions[row].tolist()) - {-1} @@ -306,7 +327,14 @@ class TestPrefillSparseIndexer(CustomTestCase): # INT32_MAX padding lands in a spare column instead of a block got = torch.zeros(rows, nb + 1, dtype=torch.bool, device="cuda") got.scatter_(1, table.blocks.clamp_max(nb).long(), True) - self.assertTrue(torch.equal(got[:, :nb], expected)) + got = got[:, :nb] + # bf16 scores tie often: blocks may differ only among those tied + # at the row's selection floor. + self.assertTrue(torch.equal(got.sum(-1), expected.sum(-1))) + scores = block_scores(case)[:, :nb] + floor = scores.masked_fill(~expected, torch.inf).amin(-1, keepdim=True) + differ = got ^ expected + self.assertTrue((~differ | (scores == floor)).all()) self.assertFalse(got[0, :nb].any(), "an empty row keeps no block") _, own_dense = publish_dense(case) for r in range(rows): @@ -320,9 +348,13 @@ class TestPrefillSparseIndexer(CustomTestCase): for rows, ctx in CASES: with self.subTest(rows=rows, ctx=ctx): case = make_case(rows, ctx, seed=rows * 3 + ctx) - got = select_sparse(publish_sparse(case)[0], case.data, case.k_cache) + table = publish_sparse(case)[0] + got = select_sparse(table, case.data, case.k_cache) want = select_dense(publish_dense(case)[0], case) - keep = reference_blocks(case).repeat_interleave(BLOCK, dim=1) + nb = -(-case.dense.shape[1] // BLOCK) + keep = torch.zeros(rows, nb + 1, dtype=torch.bool, device="cuda") + keep.scatter_(1, table.blocks.clamp_max(nb).long(), True) + keep = keep[:, :nb].repeat_interleave(BLOCK, dim=1) self.assertEqual(picks(got, 0), set(), "an empty row selects nothing") for r in range(1, rows): g, w = picks(got, r), picks(want, r) @@ -330,7 +362,17 @@ class TestPrefillSparseIndexer(CustomTestCase): self.assertTrue( keep[r, list(g)].all(), f"row {r}: outside its blocks" ) - self.assertGreaterEqual(len(g & w), MIN_OVERLAP * len(w), r) + # A dense pick the sparse side dropped at its own selection + # floor is a rounding swap, not a miss. + g_floor = case.dense[r, list(g)].min() + swaps = sum( + bool( + case.dense[r, p] + <= g_floor + g_floor.abs() * FLOOR_TOLERANCE + ) + for p in w - g + ) + self.assertGreaterEqual(len(g & w) + swaps, MIN_OVERLAP * len(w), r) floor = case.dense[r, list(w)].min() self.assertTrue( ( @@ -357,7 +399,15 @@ class TestPrefillSparseIndexer(CustomTestCase): part = select_sparse(sub, rows_of(case.data, idx), case.k_cache) for r in range(tail): a, b = picks(full, rows - tail + r), picks(part, r) - self.assertGreaterEqual(len(a & b), MIN_TAIL_OVERLAP * len(a), r) + # A full-table pick the tail dropped at its own selection floor is a + # rounding swap, as in the dense comparison above. + scores = case.dense[rows - tail + r] + b_floor = scores[list(b)].min() + swaps = sum( + bool(scores[p] <= b_floor + b_floor.abs() * FLOOR_TOLERANCE) + for p in a - b + ) + self.assertGreaterEqual(len(a & b) + swaps, MIN_TAIL_OVERLAP * len(a), r) @torch.inference_mode() def test_row_pair_ids_schedule_the_same_logits_as_per_request_ids(self):