Files
sglang/test/registered/kernels/ops/quantization/test_fp4_indexer_quant.py
T

124 lines
4.0 KiB
Python

from __future__ import annotations
import sys
import pytest
import torch
from sglang.kernels.ops.quantization.fp4_indexer_quant import (
quantize_fp4_indexer_tensor,
)
from sglang.srt.utils import get_device
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.deepseek_v4.common import quantize_fp4_indexer_ref
register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large")
HEAD_DIM = 128
FP4_DIM = HEAD_DIM // 2
GROUP_SIZE = 32
@pytest.mark.parametrize("num_tokens", [1, 7, 96])
def test_quantize_fp4_indexer_tensor(num_tokens: int) -> None:
torch.manual_seed(num_tokens)
x = torch.randn(num_tokens, HEAD_DIM, device=get_device(), dtype=torch.bfloat16)
x[0, :8] = torch.tensor(
[-8.0, -6.0, -3.0, -1.5, 0.0, 0.5, 2.0, 8.0],
device=get_device(),
dtype=torch.bfloat16,
)
x_fp4, x_sf = quantize_fp4_indexer_tensor(x)
ref_fp4, ref_sf = quantize_fp4_indexer_ref(x)
torch.testing.assert_close(x_fp4.view(torch.uint8), ref_fp4)
torch.testing.assert_close(x_sf, ref_sf)
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="Vectorized prefill dispatch targets Blackwell",
)
@pytest.mark.parametrize("rows", [4096, 4097, 16384, 524288])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("rne", [False, True])
@pytest.mark.parametrize("strided", [False, True])
def test_prefill_quantization_matches_single_row_and_replay(rows, dtype, rne, strided):
from sglang.kernels.ops.quantization.fp4_indexer_quant import (
_quantize_fp4_indexer_kernel,
)
x = torch.randn(rows, 256 if strided else 128, device="cuda", dtype=dtype)
if strided:
x = x[:, ::2]
def reference():
q = torch.empty(rows, 64, device="cuda", dtype=torch.int8)
sf = torch.empty(rows, device="cuda", dtype=torch.int32)
_quantize_fp4_indexer_kernel[(rows,)](
x.contiguous(),
q,
sf,
BLOCK_N=128,
GROUP_N=32,
RNE=rne,
)
return q, sf
for _ in range(3):
quantize_fp4_indexer_tensor(x, rne)
reference()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = quantize_fp4_indexer_tensor(x, rne)
boundaries = torch.tensor(
[0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, 6.0], device="cuda", dtype=dtype
)
for scale in (0.0, 1e-6, 1.0, 1e3):
x.normal_().mul_(scale)
x[:2] = boundaries.repeat(16)
x[1].neg_()
graph.replay()
for a, b in zip(actual, reference()):
assert torch.equal(a, b)
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="Vectorized prefill dispatch targets Blackwell",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("rne", [False, True])
def test_prefill_quantization_nonfinite_group_replay(dtype, rne):
from sglang.kernels.ops.quantization.fp4_indexer_quant import (
_quantize_fp4_indexer_kernel,
)
rows = 4097 # Also exercise the final partial CTA.
x = torch.randn(rows, HEAD_DIM, device="cuda", dtype=dtype)
expected = (
torch.empty(rows, FP4_DIM, device="cuda", dtype=torch.int8),
torch.empty(rows, device="cuda", dtype=torch.int32),
)
for _ in range(3):
quantize_fp4_indexer_tensor(x, rne)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = quantize_fp4_indexer_tensor(x, rne)
for value in (float("inf"), float("-inf"), float("nan"), 0.0):
x.normal_()
x[:, 0] = value
x[:, 32:64] = value
x[-1, 96:] = value
graph.replay()
_quantize_fp4_indexer_kernel[(rows,)](
x, *expected, BLOCK_N=HEAD_DIM, GROUP_N=GROUP_SIZE, RNE=rne
)
for a, b in zip(actual, expected):
assert torch.equal(a, b)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))