[Perf] deepseek_v4: fold the MegaAttention query layout into wq_b for block-FP8-as-MXFP8 checkpoints (#44838)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-10-17 02:45:53 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 80f40522cd
commit a126e8d640
+26 -4
View File
@@ -1377,9 +1377,28 @@ class MQALayer(MqaAttentionBase):
self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8()
)
if self.mega_wq_b_permuted:
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
param.data.copy_(interleave_query_rows(param.data, self.n_local_heads))
self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b)
method = self.wq_b.quant_method
if method.use_mxfp8:
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
param.data.copy_(
interleave_query_rows(param.data, self.n_local_heads)
)
method._process_mxfp8_linear_weight_scale(self.wq_b)
else:
block_n, block_k = method.weight_block_size
sf = self.wq_b.weight_scale_inv.data.repeat_interleave(
block_n // 16, dim=0
)
sf = (
sf.view(self.n_local_heads, 32, -1)
.transpose(0, 1)
.reshape(sf.shape)
)
self.wq_b.weight.data.copy_(
interleave_query_rows(self.wq_b.weight.data, self.n_local_heads)
)
self.wq_b.weight_scale_inv.data = sf.contiguous()
method.weight_block_size = [16, block_k]
wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1)
self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a))
@@ -1387,7 +1406,10 @@ class MQALayer(MqaAttentionBase):
method = self.wq_b.quant_method
backend = getattr(method, "mxfp8_dense_backend", None)
return bool(
getattr(method, "use_mxfp8", False)
(
getattr(method, "use_mxfp8", False)
or getattr(method, "block_fp8_as_mxfp8", False)
)
and backend is not None
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
)