[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
YAMY1234
parent
88f95f4b87
commit
eb9c9ee99d
@@ -1,5 +1,7 @@
|
||||
"""Fused QSA indexer-prep kernels must match the eager indexer path bit-for-bit,
|
||||
up to rare last-ulp RMSNorm flips (see assert_bit_comparable)."""
|
||||
up to rare last-ulp RMSNorm flips (see assert_bit_comparable). With an fp8
|
||||
indexer cache the same rows are cast to e4m3 at the store, so a bf16 last-ulp
|
||||
flip may move the e4m3 code by one step (see assert_fp8_within_one_ulp)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -85,13 +87,22 @@ class FakePool:
|
||||
|
||||
index_state_dtype = torch.bfloat16
|
||||
|
||||
def __init__(self, num_slots, num_compressed, device, dtype=torch.bfloat16):
|
||||
def __init__(
|
||||
self,
|
||||
num_slots,
|
||||
num_compressed,
|
||||
device,
|
||||
dtype=torch.bfloat16,
|
||||
compressed_dtype=None,
|
||||
):
|
||||
# Ring in the compute dtype, compressed cache and index Q in the storage dtype.
|
||||
self.qsa_compressed_dtype = compressed_dtype or dtype
|
||||
self.key_state = torch.zeros(num_slots, 1, HEAD_DIM, dtype=dtype, device=device)
|
||||
self.qsa_rope_position_buffer = torch.zeros(
|
||||
num_slots, 3, dtype=torch.int64, device=device
|
||||
)
|
||||
self.compressed = torch.zeros(
|
||||
num_compressed, 1, HEAD_DIM, dtype=dtype, device=device
|
||||
num_compressed, 1, HEAD_DIM, dtype=self.qsa_compressed_dtype, device=device
|
||||
)
|
||||
|
||||
def get_qsa_key_state_buffer(self, layer_id):
|
||||
@@ -128,6 +139,17 @@ def assert_bit_comparable(actual, expected, max_frac=1e-5, max_abs=0.02):
|
||||
assert peak <= max_abs, f"largest deviation {peak} exceeds {max_abs}"
|
||||
|
||||
|
||||
def assert_fp8_within_one_ulp(actual, expected):
|
||||
"""One e4m3 code step, or 2^-8 absolute for subnormals and sign crossings."""
|
||||
assert actual.dtype == expected.dtype == torch.float8_e4m3fn
|
||||
code_diff = (
|
||||
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
|
||||
).abs()
|
||||
abs_diff = (actual.float() - expected.float()).abs()
|
||||
bad = int((~((code_diff <= 1) | (abs_diff <= 2**-8))).sum())
|
||||
assert bad == 0, f"{bad} fp8 elements differ by more than one ulp"
|
||||
|
||||
|
||||
def _eager_compress_reference(indexer, pool, group_locs, write_locs):
|
||||
"""The pre-fusion compression chain, via the indexer's own helpers."""
|
||||
key_groups = pool.get_qsa_key_state_buffer(0)[group_locs.long()]
|
||||
@@ -144,15 +166,18 @@ def _eager_compress_reference(indexer, pool, group_locs, write_locs):
|
||||
"mrope_section, mrope_interleaved",
|
||||
[([24, 20, 20], True), ([24, 20, 20], False), (None, False)],
|
||||
)
|
||||
def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleaved):
|
||||
@pytest.mark.parametrize("compressed_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_fused_compress_matches_eager(
|
||||
num_groups, mrope_section, mrope_interleaved, compressed_dtype
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
torch.manual_seed(num_groups)
|
||||
rotary = _make_rotary(mrope_section, mrope_interleaved, device, dtype)
|
||||
indexer = _make_indexer(rotary, device, dtype)
|
||||
|
||||
pool_ref = FakePool(8192, 4096, device, dtype)
|
||||
pool_new = FakePool(8192, 4096, device, dtype)
|
||||
pool_ref = FakePool(8192, 4096, device, dtype, compressed_dtype)
|
||||
pool_new = FakePool(8192, 4096, device, dtype, compressed_dtype)
|
||||
pool_new.key_state.copy_(
|
||||
pool_ref.key_state.copy_(
|
||||
torch.randn(8192, 1, HEAD_DIM, device=device, dtype=dtype)
|
||||
@@ -172,7 +197,122 @@ def test_fused_compress_matches_eager(num_groups, mrope_section, mrope_interleav
|
||||
_eager_compress_reference(indexer, pool_ref, group_locs, write_locs)
|
||||
indexer._fused_compress_store(pool_new, group_locs, write_locs)
|
||||
|
||||
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
|
||||
if compressed_dtype == torch.float8_e4m3fn:
|
||||
assert_fp8_within_one_ulp(pool_new.compressed, pool_ref.compressed)
|
||||
else:
|
||||
assert_bit_comparable(pool_new.compressed, pool_ref.compressed)
|
||||
|
||||
|
||||
def test_fused_q_prep_fp8_matches_eager_cast():
|
||||
"""fp8 Q from the fused prep must match the eager Q cast at the end (one ulp)."""
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
torch.manual_seed(11)
|
||||
rotary = _make_rotary([24, 20, 20], True, device, dtype)
|
||||
indexer = _make_indexer(rotary, device, dtype)
|
||||
tokens = 300
|
||||
hidden = torch.randn(tokens, HIDDEN, device=device, dtype=dtype)
|
||||
positions = (
|
||||
torch.arange(5000, 5000 + tokens, device=device)
|
||||
.unsqueeze(0)
|
||||
.expand(3, -1)
|
||||
.contiguous()
|
||||
)
|
||||
qk, _ = indexer.index_qk_proj(hidden)
|
||||
q_ref = indexer.q_layernorm(qk[:, : NUM_Q_HEADS * HEAD_DIM].reshape(-1, HEAD_DIM))
|
||||
q_ref = indexer.apply_rope(positions, q_ref.reshape(tokens, NUM_Q_HEADS, HEAD_DIM))
|
||||
pool = FakePool(4096, 4096, device, dtype, torch.float8_e4m3fn)
|
||||
cache_loc = torch.arange(1, tokens + 1, device=device)
|
||||
q_new, _, stored = indexer.project_qk(
|
||||
hidden, positions, pool=pool, cache_loc=cache_loc
|
||||
)
|
||||
assert stored
|
||||
assert q_new.dtype == torch.float8_e4m3fn
|
||||
assert_fp8_within_one_ulp(q_new, q_ref.to(torch.float8_e4m3fn))
|
||||
|
||||
|
||||
def _selected_score_multisets(logits, block_indices):
|
||||
"""Sorted logits of the selected blocks per row, so tied blocks compare equal."""
|
||||
out = []
|
||||
for row in range(block_indices.shape[0]):
|
||||
picked = block_indices[row][block_indices[row] >= 0].long()
|
||||
out.append(logits[row, picked].sort().values)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_tilelang_mqa_matches_torch_reference(dtype):
|
||||
"""TileLang scoring vs the fp32 torch reference on the same operands. The
|
||||
selected blocks are compared as score multisets because fp8 ties at the
|
||||
top-k boundary."""
|
||||
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
|
||||
from sglang.srt.layers.attention.qsa.mqa import (
|
||||
HAS_TILELANG,
|
||||
tilelang_qsa_mqa_decode,
|
||||
tilelang_qsa_mqa_prefill,
|
||||
torch_qsa_mqa_decode,
|
||||
torch_qsa_mqa_prefill,
|
||||
)
|
||||
|
||||
if not HAS_TILELANG:
|
||||
pytest.skip("tilelang unavailable")
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(3)
|
||||
# Decode: paged compressed cache. qsa_fast_topk only supports the production
|
||||
# block budget (512).
|
||||
batch, max_pages, page_size, block_topk = 6, 256, 16, 512
|
||||
q = torch.randn(batch, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
|
||||
cache = torch.randn(512, page_size, 1, HEAD_DIM, device=device).to(dtype)
|
||||
page_table = torch.stack(
|
||||
[torch.randperm(512, device=device)[:max_pages] for _ in range(batch)]
|
||||
).to(torch.int32)
|
||||
context_lens = torch.randint(
|
||||
600, max_pages * page_size + 1, (batch,), device=device
|
||||
).to(torch.int32)
|
||||
max_model_len = max_pages * page_size
|
||||
ref = torch_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
|
||||
out = tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, max_model_len)
|
||||
finite = torch.isfinite(ref)
|
||||
assert torch.equal(finite, torch.isfinite(out))
|
||||
torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3)
|
||||
row_starts = torch.zeros_like(context_lens)
|
||||
sel_ref = _selected_score_multisets(
|
||||
ref, qsa_fast_topk(ref, row_starts, context_lens, topk=block_topk)
|
||||
)
|
||||
sel_out = _selected_score_multisets(
|
||||
ref, qsa_fast_topk(out, row_starts, context_lens, topk=block_topk)
|
||||
)
|
||||
for a, b in zip(sel_ref, sel_out):
|
||||
torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3)
|
||||
# Prefill: packed keys with ragged causal ranges.
|
||||
rows, keys = 70, 512
|
||||
qp = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(dtype)
|
||||
kp = torch.randn(keys, 1, HEAD_DIM, device=device).to(dtype)
|
||||
row_starts = torch.zeros(rows, dtype=torch.int32, device=device)
|
||||
row_ends = torch.randint(1, keys + 1, (rows,), device=device).to(torch.int32)
|
||||
ref_p = torch_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
|
||||
out_p = tilelang_qsa_mqa_prefill(qp, kp, row_starts, row_ends)
|
||||
finite = torch.isfinite(ref_p)
|
||||
assert torch.equal(finite, torch.isfinite(out_p))
|
||||
torch.testing.assert_close(out_p[finite], ref_p[finite], rtol=2e-3, atol=2e-3)
|
||||
|
||||
|
||||
def test_tilelang_mqa_rejects_mixed_operand_dtypes():
|
||||
"""Mixed fp8/bf16 operands must fail instead of dequantizing the cache."""
|
||||
from sglang.srt.layers.attention.qsa.mqa import (
|
||||
HAS_TILELANG,
|
||||
tilelang_qsa_mqa_decode,
|
||||
)
|
||||
|
||||
if not HAS_TILELANG:
|
||||
pytest.skip("tilelang unavailable")
|
||||
device = torch.device("cuda")
|
||||
q = torch.randn(2, NUM_Q_HEADS, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
cache = torch.randn(8, 16, 1, HEAD_DIM, device=device).to(torch.float8_e4m3fn)
|
||||
page_table = torch.zeros(2, 4, dtype=torch.int32, device=device)
|
||||
context_lens = torch.full((2,), 32, dtype=torch.int32, device=device)
|
||||
with pytest.raises(ValueError, match="same dtype"):
|
||||
tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
|
||||
|
||||
@@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase):
|
||||
[(0, 2), (1, 3)],
|
||||
)
|
||||
|
||||
def _qsa_compressed_manager(self, *, src_item_len):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.attn_tp_size = 4
|
||||
manager.pp_size = 1
|
||||
manager.is_mla_backend = False
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.kv_args = SimpleNamespace(
|
||||
engine_rank=0,
|
||||
state_types=[StateType.QSA_COMPRESSED],
|
||||
state_data_ptrs=[[1000]],
|
||||
state_item_lens=[[src_item_len]],
|
||||
state_dim_per_tensor=[[]],
|
||||
state_layer_ids=[[24]],
|
||||
)
|
||||
manager.sent = []
|
||||
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
|
||||
return manager
|
||||
|
||||
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
|
||||
"""Equal attention TP with fp8 on one peer only halves that peer's
|
||||
compressed item length. The transfer uses the source length as the
|
||||
destination stride, so the mismatch must be rejected before any write."""
|
||||
req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]])
|
||||
decode_info = SimpleNamespace(
|
||||
dst_attn_tp_size=4,
|
||||
dst_state_data_ptrs=[[2000]],
|
||||
dst_state_item_lens=[[16]],
|
||||
dst_state_dim_per_tensor=[[]],
|
||||
dst_state_layer_ids=[[24]],
|
||||
)
|
||||
manager = self._qsa_compressed_manager(src_item_len=32)
|
||||
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
|
||||
MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(manager.sent, [])
|
||||
|
||||
manager = self._qsa_compressed_manager(src_item_len=16)
|
||||
rc = MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(manager.sent), 1)
|
||||
self.assertEqual(manager.sent[0]["item_lens"], [16])
|
||||
|
||||
def test_replicated_state_tp_policy(self):
|
||||
for src_tp, dst_tp, rank, expected in (
|
||||
(4, 1, 0, True),
|
||||
|
||||
@@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self._resolved(self._construct(*qwen4), "ple_offload_embedding")
|
||||
)
|
||||
|
||||
def test_qwen4_fp8_indexer_dtype_platform_gate(self):
|
||||
"""fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16
|
||||
spellings never consult the platform."""
|
||||
qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp")
|
||||
compressed = {
|
||||
"indexer_n_heads": 4,
|
||||
"indexer_kv_heads": 1,
|
||||
"indexer_head_dim": 128,
|
||||
"indexer_budget": 2048,
|
||||
"indexer_compress_ratio": 4,
|
||||
}
|
||||
with override_platform(is_cuda=True, is_sm100=True):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3")
|
||||
# No compressed indexer fields: no QSA profile, nothing to store in fp8.
|
||||
with self.assertRaisesRegex(ValueError, "compressed QSA indexer"):
|
||||
self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3")
|
||||
with override_platform(
|
||||
is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "SM90/SM100"):
|
||||
self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
for name in ("auto", "bfloat16"):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype=name
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name)
|
||||
|
||||
def test_minimax_m2_enables_tf32_matmul(self):
|
||||
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
|
||||
self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))
|
||||
|
||||
Reference in New Issue
Block a user