[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
Jackey Hua
2026-09-29 22:11:46 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 YAMY1234
parent 88f95f4b87
commit eb9c9ee99d
13 changed files with 416 additions and 54 deletions
@@ -1,5 +1,7 @@
"""Fused QSA indexer-prep kernels must match the eager indexer path bit-for-bit,
up to rare last-ulp RMSNorm flips (see assert_bit_comparable)."""
up to rare last-ulp RMSNorm flips (see assert_bit_comparable). With an fp8
indexer cache the same rows are cast to e4m3 at the store, so a bf16 last-ulp
flip may move the e4m3 code by one step (see assert_fp8_within_one_ulp)."""
from types import SimpleNamespace
@@ -85,13 +87,22 @@ class FakePool:
index_state_dtype = torch.bfloat16
def __init__(self, num_slots, num_compressed, device, dtype=torch.bfloat16):
def __init__(
self,
num_slots,
num_compressed,
device,
dtype=torch.bfloat16,
compressed_dtype=None,
):
# Ring in the compute dtype, compressed cache and index Q in the storage dtype.
self.qsa_compressed_dtype = compressed_dtype or dtype
self.key_state = torch.zeros(num_slots, 1, HEAD_DIM, dtype=dtype, device=device)
self.qsa_rope_position_buffer = torch.zeros(
num_slots, 3, dtype=torch.int64, device=device
)
self.compressed = torch.zeros(
num_compressed, 1, HEAD_DIM, dtype=dtype, device=device
num_compressed, 1, HEAD_DIM, dtype=self.qsa_compressed_dtype, device=device
)
def get_qsa_key_state_buffer(self, layer_id):
@@ -128,6 +139,17 @@ def assert_bit_comparable(actual, expected, max_frac=1e-5, max_abs=0.02):
assert peak <= max_abs, f"largest deviation {peak} exceeds {max_abs}"
def assert_fp8_within_one_ulp(actual, expected):
"""One e4m3 code step, or 2^-8 absolute for subnormals and sign crossings."""
assert actual.dtype == expected.dtype == torch.float8_e4m3fn
code_diff = (
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
).abs()
abs_diff = (actual.float() - expected.float()).abs()
bad = int((~((code_diff <= 1) | (abs_diff <= 2**-8))).sum())
assert bad == 0, f"{bad} fp8 elements differ by more than one ulp"
def _eager_compress_reference(indexer, pool, group_locs, write_locs):
"""The pre-fusion compression chain, via the indexer's own helpers."""
key_groups = pool.get_qsa_key_state_buffer(0)[group_locs.long()]
@@ -144,15 +166,18 @@ def _eager_compress_reference(indexer, pool, group_locs, write_locs):
"mrope_section, mrope_interleaved",
[([24, 20, 20], True), ([24, 20, 20], False), (None, False)],
)
def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleaved):
@pytest.mark.parametrize("compressed_dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_fused_compress_matches_eager(
num_groups, mrope_section, mrope_interleaved, compressed_dtype
):
device = torch.device("cuda")
dtype = torch.bfloat16
torch.manual_seed(num_groups)
rotary = _make_rotary(mrope_section, mrope_interleaved, device, dtype)
indexer = _make_indexer(rotary, device, dtype)
pool_ref = FakePool(8192, 4096, device, dtype)
pool_new = FakePool(8192, 4096, device, dtype)
pool_ref = FakePool(8192, 4096, device, dtype, compressed_dtype)
pool_new = FakePool(8192, 4096, device, dtype, compressed_dtype)
pool_new.key_state.copy_(
pool_ref.key_state.copy_(
torch.randn(8192, 1, HEAD_DIM, device=device, dtype=dtype)
@@ -172,7 +197,122 @@ def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleav
_eager_compress_reference(indexer, pool_ref, group_locs, write_locs)
indexer._fused_compress_store(pool_new, group_locs, write_locs)
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
if compressed_dtype == torch.float8_e4m3fn:
assert_fp8_within_one_ulp(pool_new.compressed, pool_ref.compressed)
else:
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
def test_fused_q_prep_fp8_matches_eager_cast():
"""fp8 Q from the fused prep must match the eager Q cast at the end (one ulp)."""
device = torch.device("cuda")
dtype = torch.bfloat16
torch.manual_seed(11)
rotary = _make_rotary([24, 20, 20], True, device, dtype)
indexer = _make_indexer(rotary, device, dtype)
tokens = 300
hidden = torch.randn(tokens, HIDDEN, device=device, dtype=dtype)
positions = (
torch.arange(5000, 5000 + tokens, device=device)
.unsqueeze(0)
.expand(3, -1)
.contiguous()
)
qk, _ = indexer.index_qk_proj(hidden)
q_ref = indexer.q_layernorm(qk[:, : NUM_Q_HEADS * HEAD_DIM].reshape(-1, HEAD_DIM))
q_ref = indexer.apply_rope(positions, q_ref.reshape(tokens, NUM_Q_HEADS, HEAD_DIM))
pool = FakePool(4096, 4096, device, dtype, torch.float8_e4m3fn)
cache_loc = torch.arange(1, tokens + 1, device=device)
q_new, _, stored = indexer.project_qk(
hidden, positions, pool=pool, cache_loc=cache_loc
)
assert stored
assert q_new.dtype == torch.float8_e4m3fn
assert_fp8_within_one_ulp(q_new, q_ref.to(torch.float8_e4m3fn))
def _selected_score_multisets(logits, block_indices):
"""Sorted logits of the selected blocks per row, so tied blocks compare equal."""
out = []
for row in range(block_indices.shape[0]):
picked = block_indices[row][block_indices[row] >= 0].long()
out.append(logits[row, picked].sort().values)
return out
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_tilelang_mqa_matches_torch_reference(dtype):
"""TileLang scoring vs the fp32 torch reference on the same operands. The
selected blocks are compared as score multisets because fp8 ties at the
top-k boundary."""
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
from sglang.srt.layers.attention.qsa.mqa import (
HAS_TILELANG,
tilelang_qsa_mqa_decode,
tilelang_qsa_mqa_prefill,
torch_qsa_mqa_decode,
torch_qsa_mqa_prefill,
)
if not HAS_TILELANG:
pytest.skip("tilelang unavailable")
device = torch.device("cuda")
torch.manual_seed(3)
# Decode: paged compressed cache. qsa_fast_topk only supports the production
# block budget (512).
batch, max_pages, page_size, block_topk = 6, 256, 16, 512
q = torch.randn(batch, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
cache = torch.randn(512, page_size, 1, HEAD_DIM, device=device).to(dtype)
page_table = torch.stack(
[torch.randperm(512, device=device)[:max_pages] for _ in range(batch)]
).to(torch.int32)
context_lens = torch.randint(
600, max_pages * page_size + 1, (batch,), device=device
).to(torch.int32)
max_model_len = max_pages * page_size
ref = torch_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
out = tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
finite = torch.isfinite(ref)
assert torch.equal(finite, torch.isfinite(out))
torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3)
row_starts = torch.zeros_like(context_lens)
sel_ref = _selected_score_multisets(
ref, qsa_fast_topk(ref, row_starts, context_lens, topk=block_topk)
)
sel_out = _selected_score_multisets(
ref, qsa_fast_topk(out, row_starts, context_lens, topk=block_topk)
)
for a, b in zip(sel_ref, sel_out):
torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3)
# Prefill: packed keys with ragged causal ranges.
rows, keys = 70, 512
qp = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
kp = torch.randn(keys, 1, HEAD_DIM, device=device).to(dtype)
row_starts = torch.zeros(rows, dtype=torch.int32, device=device)
row_ends = torch.randint(1, keys + 1, (rows,), device=device).to(torch.int32)
ref_p = torch_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
out_p = tilelang_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
finite = torch.isfinite(ref_p)
assert torch.equal(finite, torch.isfinite(out_p))
torch.testing.assert_close(out_p[finite], ref_p[finite], rtol=2e-3, atol=2e-3)
def test_tilelang_mqa_rejects_mixed_operand_dtypes():
"""Mixed fp8/bf16 operands must fail instead of dequantizing the cache."""
from sglang.srt.layers.attention.qsa.mqa import (
HAS_TILELANG,
tilelang_qsa_mqa_decode,
)
if not HAS_TILELANG:
pytest.skip("tilelang unavailable")
device = torch.device("cuda")
q = torch.randn(2, NUM_Q_HEADS, HEAD_DIM, device=device, dtype=torch.bfloat16)
cache = torch.randn(8, 16, 1, HEAD_DIM, device=device).to(torch.float8_e4m3fn)
page_table = torch.zeros(2, 4, dtype=torch.int32, device=device)
context_lens = torch.full((2,), 32, dtype=torch.int32, device=device)
with pytest.raises(ValueError, match="same dtype"):
tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64)
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])