[Perf] dsv4: Score the V4.1 index logits in bf16 on SM100 (#45880)
This commit is contained in:
@@ -4,6 +4,7 @@ memory budget), the paged pool, or the published blocks of a sparse table."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import Iterator, Tuple
|
||||
|
||||
import torch
|
||||
@@ -17,10 +18,34 @@ from sglang.srt.utils.common import ceil_align
|
||||
|
||||
from .candidate_table import CANDIDATE_BLOCK_SIZE
|
||||
|
||||
_LOGITS_DTYPE = (
|
||||
torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32
|
||||
)
|
||||
_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize
|
||||
|
||||
@functools.cache
|
||||
def _is_sm100() -> bool:
|
||||
return (
|
||||
torch.cuda.is_available()
|
||||
and torch.version.cuda is not None
|
||||
and torch.cuda.get_device_capability()[0] == 10
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def index_logits_dtype() -> torch.dtype:
|
||||
"""bf16 on SM100, where the reference kernels score in bf16; fp32 elsewhere.
|
||||
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS, when set, overrides the choice."""
|
||||
forced = envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get()
|
||||
bf16 = _is_sm100() if forced is None else forced
|
||||
return torch.bfloat16 if bf16 else torch.float32
|
||||
|
||||
|
||||
@functools.cache
|
||||
def index_weights_dtype() -> torch.dtype:
|
||||
"""DeepGEMM reduces in the head weights' dtype and takes bf16 weights only on
|
||||
SM100 with bf16 logits."""
|
||||
if index_logits_dtype() == torch.bfloat16 and _is_sm100():
|
||||
return torch.bfloat16
|
||||
return torch.float32
|
||||
|
||||
|
||||
# Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which
|
||||
# DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row)
|
||||
FLAT_LOGITS_WIDTH_ALIGN = 64
|
||||
@@ -32,16 +57,15 @@ def flat_index_logits_rows_per_tile(
|
||||
*,
|
||||
heads: int,
|
||||
budget_bytes: int,
|
||||
item_size: int = _LOGITS_DTYPE_BYTES,
|
||||
) -> int:
|
||||
"""Rows per logits tile within ``budget_bytes`` (sized as fp32, which also
|
||||
bounds bf16), at the kernel's row alignment."""
|
||||
"""Rows per logits tile within ``budget_bytes``, at the kernel's row alignment.
|
||||
Tiles are sized as fp32 logits whatever the logits dtype: bf16 tiles then fill
|
||||
half the budget, which leaves room for the per-row selection scratch that
|
||||
grows with the tile."""
|
||||
row_alignment = 128 // heads
|
||||
# NOTE: for bf16, the estimated row bytes is 2 times the actual bytes
|
||||
effective_width = width * item_size // 4
|
||||
rows_per_chunk = mqa_logits_rows_per_chunk(
|
||||
num_rows=ceil_align(rows, row_alignment),
|
||||
row_bytes=mqa_logits_row_bytes(effective_width),
|
||||
row_bytes=mqa_logits_row_bytes(width),
|
||||
budget_bytes=budget_bytes,
|
||||
)
|
||||
if rows_per_chunk is None:
|
||||
@@ -72,6 +96,8 @@ def flat_index_logits_tiles(
|
||||
rows_per_chunk = flat_index_logits_rows_per_tile(
|
||||
rows, width, heads=q[0].shape[1], budget_bytes=budget_bytes
|
||||
)
|
||||
weights = weights.to(index_weights_dtype())
|
||||
logits_dtype = index_logits_dtype()
|
||||
for offset in range(0, rows, rows_per_chunk):
|
||||
tile = slice(offset, min(offset + rows_per_chunk, rows))
|
||||
tile_starts = starts[tile]
|
||||
@@ -85,7 +111,7 @@ def flat_index_logits_tiles(
|
||||
tile_starts + lengths[tile],
|
||||
False,
|
||||
width,
|
||||
logits_dtype=_LOGITS_DTYPE,
|
||||
logits_dtype=logits_dtype,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -108,13 +134,13 @@ def deep_gemm_fp4_paged_mqa_logits(
|
||||
return fp8_fp4_paged_mqa_logits(
|
||||
q_fp4,
|
||||
k_cache,
|
||||
weights,
|
||||
weights.to(index_weights_dtype()),
|
||||
sl,
|
||||
page_table,
|
||||
deep_gemm_metadata,
|
||||
max_seq_len,
|
||||
False,
|
||||
logits_dtype=_LOGITS_DTYPE,
|
||||
logits_dtype=index_logits_dtype(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1748,7 +1748,8 @@ class Envs:
|
||||
SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32")
|
||||
# DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead
|
||||
# of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16.
|
||||
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False)
|
||||
# Unset: bf16 on SM100 (with bf16 head weights), fp32 elsewhere.
|
||||
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(None)
|
||||
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
|
||||
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
|
||||
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16
|
||||
|
||||
Reference in New Issue
Block a user