[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
Jackey Hua
2026-09-29 22:11:46 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 YAMY1234
parent 88f95f4b87
commit eb9c9ee99d
13 changed files with 416 additions and 54 deletions
@@ -1,5 +1,7 @@
"""Fused QSA indexer-prep kernels must match the eager indexer path bit-for-bit,
up to rare last-ulp RMSNorm flips (see assert_bit_comparable)."""
up to rare last-ulp RMSNorm flips (see assert_bit_comparable). With an fp8
indexer cache the same rows are cast to e4m3 at the store, so a bf16 last-ulp
flip may move the e4m3 code by one step (see assert_fp8_within_one_ulp)."""
from types import SimpleNamespace
@@ -85,13 +87,22 @@ class FakePool:
index_state_dtype = torch.bfloat16
def __init__(self, num_slots, num_compressed, device, dtype=torch.bfloat16):
def __init__(
self,
num_slots,
num_compressed,
device,
dtype=torch.bfloat16,
compressed_dtype=None,
):
# Ring in the compute dtype, compressed cache and index Q in the storage dtype.
self.qsa_compressed_dtype = compressed_dtype or dtype
self.key_state = torch.zeros(num_slots, 1, HEAD_DIM, dtype=dtype, device=device)
self.qsa_rope_position_buffer = torch.zeros(
num_slots, 3, dtype=torch.int64, device=device
)
self.compressed = torch.zeros(
num_compressed, 1, HEAD_DIM, dtype=dtype, device=device
num_compressed, 1, HEAD_DIM, dtype=self.qsa_compressed_dtype, device=device
)
def get_qsa_key_state_buffer(self, layer_id):
@@ -128,6 +139,17 @@ def assert_bit_comparable(actual, expected, max_frac=1e-5, max_abs=0.02):
assert peak <= max_abs, f"largest deviation {peak} exceeds {max_abs}"
def assert_fp8_within_one_ulp(actual, expected):
"""One e4m3 code step, or 2^-8 absolute for subnormals and sign crossings."""
assert actual.dtype == expected.dtype == torch.float8_e4m3fn
code_diff = (
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
).abs()
abs_diff = (actual.float() - expected.float()).abs()
bad = int((~((code_diff <= 1) | (abs_diff <= 2**-8))).sum())
assert bad == 0, f"{bad} fp8 elements differ by more than one ulp"
def _eager_compress_reference(indexer, pool, group_locs, write_locs):
"""The pre-fusion compression chain, via the indexer's own helpers."""
key_groups = pool.get_qsa_key_state_buffer(0)[group_locs.long()]
@@ -144,15 +166,18 @@ def _eager_compress_reference(indexer, pool, group_locs, write_locs):
"mrope_section, mrope_interleaved",
[([24, 20, 20], True), ([24, 20, 20], False), (None, False)],
)
def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleaved):
@pytest.mark.parametrize("compressed_dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_fused_compress_matches_eager(
num_groups, mrope_section, mrope_interleaved, compressed_dtype
):
device = torch.device("cuda")
dtype = torch.bfloat16
torch.manual_seed(num_groups)
rotary = _make_rotary(mrope_section, mrope_interleaved, device, dtype)
indexer = _make_indexer(rotary, device, dtype)
pool_ref = FakePool(8192, 4096, device, dtype)
pool_new = FakePool(8192, 4096, device, dtype)
pool_ref = FakePool(8192, 4096, device, dtype, compressed_dtype)
pool_new = FakePool(8192, 4096, device, dtype, compressed_dtype)
pool_new.key_state.copy_(
pool_ref.key_state.copy_(
torch.randn(8192, 1, HEAD_DIM, device=device, dtype=dtype)
@@ -172,7 +197,122 @@ def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleav
_eager_compress_reference(indexer, pool_ref, group_locs, write_locs)
indexer._fused_compress_store(pool_new, group_locs, write_locs)
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
if compressed_dtype == torch.float8_e4m3fn:
assert_fp8_within_one_ulp(pool_new.compressed, pool_ref.compressed)
else:
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
def test_fused_q_prep_fp8_matches_eager_cast():
"""fp8 Q from the fused prep must match the eager Q cast at the end (one ulp)."""
device = torch.device("cuda")
dtype = torch.bfloat16
torch.manual_seed(11)
rotary = _make_rotary([24, 20, 20], True, device, dtype)
indexer = _make_indexer(rotary, device, dtype)
tokens = 300
hidden = torch.randn(tokens, HIDDEN, device=device, dtype=dtype)
positions = (
torch.arange(5000, 5000 + tokens, device=device)
.unsqueeze(0)
.expand(3, -1)
.contiguous()
)
qk, _ = indexer.index_qk_proj(hidden)
q_ref = indexer.q_layernorm(qk[:, : NUM_Q_HEADS * HEAD_DIM].reshape(-1, HEAD_DIM))
q_ref = indexer.apply_rope(positions, q_ref.reshape(tokens, NUM_Q_HEADS, HEAD_DIM))
pool = FakePool(4096, 4096, device, dtype, torch.float8_e4m3fn)
cache_loc = torch.arange(1, tokens + 1, device=device)
q_new, _, stored = indexer.project_qk(
hidden, positions, pool=pool, cache_loc=cache_loc
)
assert stored
assert q_new.dtype == torch.float8_e4m3fn
assert_fp8_within_one_ulp(q_new, q_ref.to(torch.float8_e4m3fn))
def _selected_score_multisets(logits, block_indices):
"""Sorted logits of the selected blocks per row, so tied blocks compare equal."""
out = []
for row in range(block_indices.shape[0]):
picked = block_indices[row][block_indices[row] >= 0].long()
out.append(logits[row, picked].sort().values)
return out
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_tilelang_mqa_matches_torch_reference(dtype):
"""TileLang scoring vs the fp32 torch reference on the same operands. The
selected blocks are compared as score multisets because fp8 ties at the
top-k boundary."""
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
from sglang.srt.layers.attention.qsa.mqa import (
HAS_TILELANG,
tilelang_qsa_mqa_decode,
tilelang_qsa_mqa_prefill,
torch_qsa_mqa_decode,
torch_qsa_mqa_prefill,
)
if not HAS_TILELANG:
pytest.skip("tilelang unavailable")
device = torch.device("cuda")
torch.manual_seed(3)
# Decode: paged compressed cache. qsa_fast_topk only supports the production
# block budget (512).
batch, max_pages, page_size, block_topk = 6, 256, 16, 512
q = torch.randn(batch, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
cache = torch.randn(512, page_size, 1, HEAD_DIM, device=device).to(dtype)
page_table = torch.stack(
[torch.randperm(512, device=device)[:max_pages] for _ in range(batch)]
).to(torch.int32)
context_lens = torch.randint(
600, max_pages * page_size + 1, (batch,), device=device
).to(torch.int32)
max_model_len = max_pages * page_size
ref = torch_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
out = tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
finite = torch.isfinite(ref)
assert torch.equal(finite, torch.isfinite(out))
torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3)
row_starts = torch.zeros_like(context_lens)
sel_ref = _selected_score_multisets(
ref, qsa_fast_topk(ref, row_starts, context_lens, topk=block_topk)
)
sel_out = _selected_score_multisets(
ref, qsa_fast_topk(out, row_starts, context_lens, topk=block_topk)
)
for a, b in zip(sel_ref, sel_out):
torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3)
# Prefill: packed keys with ragged causal ranges.
rows, keys = 70, 512
qp = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
kp = torch.randn(keys, 1, HEAD_DIM, device=device).to(dtype)
row_starts = torch.zeros(rows, dtype=torch.int32, device=device)
row_ends = torch.randint(1, keys + 1, (rows,), device=device).to(torch.int32)
ref_p = torch_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
out_p = tilelang_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
finite = torch.isfinite(ref_p)
assert torch.equal(finite, torch.isfinite(out_p))
torch.testing.assert_close(out_p[finite], ref_p[finite], rtol=2e-3, atol=2e-3)
def test_tilelang_mqa_rejects_mixed_operand_dtypes():
"""Mixed fp8/bf16 operands must fail instead of dequantizing the cache."""
from sglang.srt.layers.attention.qsa.mqa import (
HAS_TILELANG,
tilelang_qsa_mqa_decode,
)
if not HAS_TILELANG:
pytest.skip("tilelang unavailable")
device = torch.device("cuda")
q = torch.randn(2, NUM_Q_HEADS, HEAD_DIM, device=device, dtype=torch.bfloat16)
cache = torch.randn(8, 16, 1, HEAD_DIM, device=device).to(torch.float8_e4m3fn)
page_table = torch.zeros(2, 4, dtype=torch.int32, device=device)
context_lens = torch.full((2,), 32, dtype=torch.int32, device=device)
with pytest.raises(ValueError, match="same dtype"):
tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64)
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
@@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase):
[(0, 2), (1, 3)],
)
def _qsa_compressed_manager(self, *, src_item_len):
manager = object.__new__(MooncakeKVManager)
manager.attn_tp_size = 4
manager.pp_size = 1
manager.is_mla_backend = False
manager.is_hybrid_mla_backend = False
manager.kv_args = SimpleNamespace(
engine_rank=0,
state_types=[StateType.QSA_COMPRESSED],
state_data_ptrs=[[1000]],
state_item_lens=[[src_item_len]],
state_dim_per_tensor=[[]],
state_layer_ids=[[24]],
)
manager.sent = []
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
return manager
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
"""Equal attention TP with fp8 on one peer only halves that peer's
compressed item length. The transfer uses the source length as the
destination stride, so the mismatch must be rejected before any write."""
req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]])
decode_info = SimpleNamespace(
dst_attn_tp_size=4,
dst_state_data_ptrs=[[2000]],
dst_state_item_lens=[[16]],
dst_state_dim_per_tensor=[[]],
dst_state_layer_ids=[[24]],
)
manager = self._qsa_compressed_manager(src_item_len=32)
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
MooncakeKVManager.maybe_send_extra(
manager, req, [[3]], None, target_rank_registration_info=decode_info
)
self.assertEqual(manager.sent, [])
manager = self._qsa_compressed_manager(src_item_len=16)
rc = MooncakeKVManager.maybe_send_extra(
manager, req, [[3]], None, target_rank_registration_info=decode_info
)
self.assertEqual(rc, 0)
self.assertEqual(len(manager.sent), 1)
self.assertEqual(manager.sent[0]["item_lens"], [16])
def test_replicated_state_tp_policy(self):
for src_tp, dst_tp, rank, expected in (
(4, 1, 0, True),
@@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self._resolved(self._construct(*qwen4), "ple_offload_embedding")
)
def test_qwen4_fp8_indexer_dtype_platform_gate(self):
"""fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16
spellings never consult the platform."""
qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp")
compressed = {
"indexer_n_heads": 4,
"indexer_kv_heads": 1,
"indexer_head_dim": 128,
"indexer_budget": 2048,
"indexer_compress_ratio": 4,
}
with override_platform(is_cuda=True, is_sm100=True):
sa = self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
)
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3")
# No compressed indexer fields: no QSA profile, nothing to store in fp8.
with self.assertRaisesRegex(ValueError, "compressed QSA indexer"):
self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3")
with override_platform(
is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False
):
with self.assertRaisesRegex(ValueError, "SM90/SM100"):
self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
)
for name in ("auto", "bfloat16"):
sa = self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype=name
)
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name)
def test_minimax_m2_enables_tf32_matmul(self):
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))