[Qwen4-Exp] Optional fp8 (e4m3) storage for the compressed QSA indexer cache (#39614)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
YAMY1234
parent
88f95f4b87
commit
eb9c9ee99d
@@ -458,6 +458,51 @@ class TestQwen4StateWire(unittest.TestCase):
|
||||
[(0, 2), (1, 3)],
|
||||
)
|
||||
|
||||
def _qsa_compressed_manager(self, *, src_item_len):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.attn_tp_size = 4
|
||||
manager.pp_size = 1
|
||||
manager.is_mla_backend = False
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.kv_args = SimpleNamespace(
|
||||
engine_rank=0,
|
||||
state_types=[StateType.QSA_COMPRESSED],
|
||||
state_data_ptrs=[[1000]],
|
||||
state_item_lens=[[src_item_len]],
|
||||
state_dim_per_tensor=[[]],
|
||||
state_layer_ids=[[24]],
|
||||
)
|
||||
manager.sent = []
|
||||
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
|
||||
return manager
|
||||
|
||||
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
|
||||
"""Equal attention TP with fp8 on one peer only halves that peer's
|
||||
compressed item length. The transfer uses the source length as the
|
||||
destination stride, so the mismatch must be rejected before any write."""
|
||||
req = SimpleNamespace(mooncake_session_id="session", dst_state_indices=[[3]])
|
||||
decode_info = SimpleNamespace(
|
||||
dst_attn_tp_size=4,
|
||||
dst_state_data_ptrs=[[2000]],
|
||||
dst_state_item_lens=[[16]],
|
||||
dst_state_dim_per_tensor=[[]],
|
||||
dst_state_layer_ids=[[24]],
|
||||
)
|
||||
manager = self._qsa_compressed_manager(src_item_len=32)
|
||||
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
|
||||
MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(manager.sent, [])
|
||||
|
||||
manager = self._qsa_compressed_manager(src_item_len=16)
|
||||
rc = MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(manager.sent), 1)
|
||||
self.assertEqual(manager.sent[0]["item_lens"], [16])
|
||||
|
||||
def test_replicated_state_tp_policy(self):
|
||||
for src_tp, dst_tp, rank, expected in (
|
||||
(4, 1, 0, True),
|
||||
|
||||
@@ -929,6 +929,38 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self._resolved(self._construct(*qwen4), "ple_offload_embedding")
|
||||
)
|
||||
|
||||
def test_qwen4_fp8_indexer_dtype_platform_gate(self):
|
||||
"""fp8_e4m3 needs CUDA SM90/SM100 and a compressed QSA indexer. The bf16
|
||||
spellings never consult the platform."""
|
||||
qwen4 = ("Qwen4ExpForConditionalGeneration", "qwen4_exp")
|
||||
compressed = {
|
||||
"indexer_n_heads": 4,
|
||||
"indexer_kv_heads": 1,
|
||||
"indexer_head_dim": 128,
|
||||
"indexer_budget": 2048,
|
||||
"indexer_compress_ratio": 4,
|
||||
}
|
||||
with override_platform(is_cuda=True, is_sm100=True):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), "fp8_e4m3")
|
||||
# No compressed indexer fields: no QSA profile, nothing to store in fp8.
|
||||
with self.assertRaisesRegex(ValueError, "compressed QSA indexer"):
|
||||
self._construct(*qwen4, qsa_indexer_dtype="fp8_e4m3")
|
||||
with override_platform(
|
||||
is_cuda=False, is_hip=True, is_sm90=False, is_sm100=False
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "SM90/SM100"):
|
||||
self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype="fp8_e4m3"
|
||||
)
|
||||
for name in ("auto", "bfloat16"):
|
||||
sa = self._construct(
|
||||
*qwen4, config_extra=compressed, qsa_indexer_dtype=name
|
||||
)
|
||||
self.assertEqual(self._resolved(sa, "qsa_indexer_dtype"), name)
|
||||
|
||||
def test_minimax_m2_enables_tf32_matmul(self):
|
||||
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
|
||||
self.assertTrue(self._resolved(sa, "enable_tf32_matmul"))
|
||||
|
||||
Reference in New Issue
Block a user