diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index b8648f773c..19b5de5ee2 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1377,9 +1377,28 @@ class MQALayer(MqaAttentionBase): self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8() ) if self.mega_wq_b_permuted: - for param in (self.wq_b.weight, self.wq_b.weight_scale_inv): - param.data.copy_(interleave_query_rows(param.data, self.n_local_heads)) - self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b) + method = self.wq_b.quant_method + if method.use_mxfp8: + for param in (self.wq_b.weight, self.wq_b.weight_scale_inv): + param.data.copy_( + interleave_query_rows(param.data, self.n_local_heads) + ) + method._process_mxfp8_linear_weight_scale(self.wq_b) + else: + block_n, block_k = method.weight_block_size + sf = self.wq_b.weight_scale_inv.data.repeat_interleave( + block_n // 16, dim=0 + ) + sf = ( + sf.view(self.n_local_heads, 32, -1) + .transpose(0, 1) + .reshape(sf.shape) + ) + self.wq_b.weight.data.copy_( + interleave_query_rows(self.wq_b.weight.data, self.n_local_heads) + ) + self.wq_b.weight_scale_inv.data = sf.contiguous() + method.weight_block_size = [16, block_k] wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a)) @@ -1387,7 +1406,10 @@ class MQALayer(MqaAttentionBase): method = self.wq_b.quant_method backend = getattr(method, "mxfp8_dense_backend", None) return bool( - getattr(method, "use_mxfp8", False) + ( + getattr(method, "use_mxfp8", False) + or getattr(method, "block_fp8_as_mxfp8", False) + ) and backend is not None and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl()) )