[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
Jackey Hua
2026-09-29 22:11:46 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 YAMY1234
parent 88f95f4b87
commit eb9c9ee99d
13 changed files with 416 additions and 54 deletions
@@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase):
[(0, 2), (1, 3)],
)
def _qsa_compressed_manager(self, *, src_item_len):
manager = object.__new__(MooncakeKVManager)
manager.attn_tp_size = 4
manager.pp_size = 1
manager.is_mla_backend = False
manager.is_hybrid_mla_backend = False
manager.kv_args = SimpleNamespace(
engine_rank=0,
state_types=[StateType.QSA_COMPRESSED],
state_data_ptrs=[[1000]],
state_item_lens=[[src_item_len]],
state_dim_per_tensor=[[]],
state_layer_ids=[[24]],
)
manager.sent = []
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
return manager
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
"""Equal attention TP with fp8 on one peer only halves that peer's
compressed item length. The transfer uses the source length as the
destination stride, so the mismatch must be rejected before any write."""
req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]])
decode_info = SimpleNamespace(
dst_attn_tp_size=4,
dst_state_data_ptrs=[[2000]],
dst_state_item_lens=[[16]],
dst_state_dim_per_tensor=[[]],
dst_state_layer_ids=[[24]],
)
manager = self._qsa_compressed_manager(src_item_len=32)
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
MooncakeKVManager.maybe_send_extra(
manager, req, [[3]], None, target_rank_registration_info=decode_info
)
self.assertEqual(manager.sent, [])
manager = self._qsa_compressed_manager(src_item_len=16)
rc = MooncakeKVManager.maybe_send_extra(
manager, req, [[3]], None, target_rank_registration_info=decode_info
)
self.assertEqual(rc, 0)
self.assertEqual(len(manager.sent), 1)
self.assertEqual(manager.sent[0]["item_lens"], [16])
def test_replicated_state_tp_policy(self):
for src_tp, dst_tp, rank, expected in (
(4, 1, 0, True),
@@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self._resolved(self._construct(*qwen4), "ple_offload_embedding")
)
def test_qwen4_fp8_indexer_dtype_platform_gate(self):
"""fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16
spellings never consult the platform."""
qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp")
compressed = {
"indexer_n_heads": 4,
"indexer_kv_heads": 1,
"indexer_head_dim": 128,
"indexer_budget": 2048,
"indexer_compress_ratio": 4,
}
with override_platform(is_cuda=True, is_sm100=True):
sa = self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
)
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3")
# No compressed indexer fields: no QSA profile, nothing to store in fp8.
with self.assertRaisesRegex(ValueError, "compressed QSA indexer"):
self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3")
with override_platform(
is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False
):
with self.assertRaisesRegex(ValueError, "SM90/SM100"):
self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
)
for name in ("auto", "bfloat16"):
sa = self._construct(
*qwen4, config_extra=compressed, qsa_indexer_dtype=name
)
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name)
def test_minimax_m2_enables_tf32_matmul(self):
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))