[Perf] dsv4: Score the V4.1 index logits in bf16 on SM100 (#45880)

This commit is contained in:
Liangsheng Yin
2026-10-22 14:29:08 -07:00
committed by GitHub
parent 25ad569ff9
commit 840a6313c9
4 changed files with 107 additions and 23 deletions
@@ -4,6 +4,7 @@ memory budget), the paged pool, or the published blocks of a sparse table."""
from __future__ import annotations
import functools
from typing import Iterator, Tuple
import torch
@@ -17,10 +18,34 @@ from sglang.srt.utils.common import ceil_align
from .candidate_table import CANDIDATE_BLOCK_SIZE
_LOGITS_DTYPE = (
torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32
)
_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize
@functools.cache
def _is_sm100() -> bool:
return (
torch.cuda.is_available()
and torch.version.cuda is not None
and torch.cuda.get_device_capability()[0] == 10
)
@functools.cache
def index_logits_dtype() -> torch.dtype:
"""bf16 on SM100, where the reference kernels score in bf16; fp32 elsewhere.
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS, when set, overrides the choice."""
forced = envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get()
bf16 = _is_sm100() if forced is None else forced
return torch.bfloat16 if bf16 else torch.float32
@functools.cache
def index_weights_dtype() -> torch.dtype:
"""DeepGEMM reduces in the head weights' dtype and takes bf16 weights only on
SM100 with bf16 logits."""
if index_logits_dtype() == torch.bfloat16 and _is_sm100():
return torch.bfloat16
return torch.float32
# Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which
# DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row)
FLAT_LOGITS_WIDTH_ALIGN = 64
@@ -32,16 +57,15 @@ def flat_index_logits_rows_per_tile(
*,
heads: int,
budget_bytes: int,
item_size: int = _LOGITS_DTYPE_BYTES,
) -> int:
"""Rows per logits tile within ``budget_bytes`` (sized as fp32, which also
bounds bf16), at the kernel's row alignment."""
"""Rows per logits tile within ``budget_bytes``, at the kernel's row alignment.
Tiles are sized as fp32 logits whatever the logits dtype: bf16 tiles then fill
half the budget, which leaves room for the per-row selection scratch that
grows with the tile."""
row_alignment = 128 // heads
# NOTE: for bf16, the estimated row bytes is 2 times the actual bytes
effective_width = width * item_size // 4
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(rows, row_alignment),
row_bytes=mqa_logits_row_bytes(effective_width),
row_bytes=mqa_logits_row_bytes(width),
budget_bytes=budget_bytes,
)
if rows_per_chunk is None:
@@ -72,6 +96,8 @@ def flat_index_logits_tiles(
rows_per_chunk = flat_index_logits_rows_per_tile(
rows, width, heads=q[0].shape[1], budget_bytes=budget_bytes
)
weights = weights.to(index_weights_dtype())
logits_dtype = index_logits_dtype()
for offset in range(0, rows, rows_per_chunk):
tile = slice(offset, min(offset + rows_per_chunk, rows))
tile_starts = starts[tile]
@@ -85,7 +111,7 @@ def flat_index_logits_tiles(
tile_starts + lengths[tile],
False,
width,
logits_dtype=_LOGITS_DTYPE,
logits_dtype=logits_dtype,
),
)
@@ -108,13 +134,13 @@ def deep_gemm_fp4_paged_mqa_logits(
return fp8_fp4_paged_mqa_logits(
q_fp4,
k_cache,
weights,
weights.to(index_weights_dtype()),
sl,
page_table,
deep_gemm_metadata,
max_seq_len,
False,
logits_dtype=_LOGITS_DTYPE,
logits_dtype=index_logits_dtype(),
)
+2 -1
View File
@@ -1748,7 +1748,8 @@ class Envs:
SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32")
# DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead
# of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16.
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False)
# Unset: bf16 on SM100 (with bf16 head weights), fp32 elsewhere.
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(None)
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16