[Perf] deepseek_v4: fold the MegaAttention query layout into wq_b for block-FP8-as-MXFP8 checkpoints (#44838)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
80f40522cd
commit
a126e8d640
@@ -1377,9 +1377,28 @@ class MQALayer(MqaAttentionBase):
|
||||
self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8()
|
||||
)
|
||||
if self.mega_wq_b_permuted:
|
||||
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
|
||||
param.data.copy_(interleave_query_rows(param.data, self.n_local_heads))
|
||||
self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b)
|
||||
method = self.wq_b.quant_method
|
||||
if method.use_mxfp8:
|
||||
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
|
||||
param.data.copy_(
|
||||
interleave_query_rows(param.data, self.n_local_heads)
|
||||
)
|
||||
method._process_mxfp8_linear_weight_scale(self.wq_b)
|
||||
else:
|
||||
block_n, block_k = method.weight_block_size
|
||||
sf = self.wq_b.weight_scale_inv.data.repeat_interleave(
|
||||
block_n // 16, dim=0
|
||||
)
|
||||
sf = (
|
||||
sf.view(self.n_local_heads, 32, -1)
|
||||
.transpose(0, 1)
|
||||
.reshape(sf.shape)
|
||||
)
|
||||
self.wq_b.weight.data.copy_(
|
||||
interleave_query_rows(self.wq_b.weight.data, self.n_local_heads)
|
||||
)
|
||||
self.wq_b.weight_scale_inv.data = sf.contiguous()
|
||||
method.weight_block_size = [16, block_k]
|
||||
wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1)
|
||||
self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a))
|
||||
|
||||
@@ -1387,7 +1406,10 @@ class MQALayer(MqaAttentionBase):
|
||||
method = self.wq_b.quant_method
|
||||
backend = getattr(method, "mxfp8_dense_backend", None)
|
||||
return bool(
|
||||
getattr(method, "use_mxfp8", False)
|
||||
(
|
||||
getattr(method, "use_mxfp8", False)
|
||||
or getattr(method, "block_fp8_as_mxfp8", False)
|
||||
)
|
||||
and backend is not None
|
||||
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user