[Perf] deepseek_v4: fuse block-FP8 shared experts into MegaMoE for any 32-aligned block size (#44837)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
9c7b338a19
commit
14c4dda24e
@@ -200,7 +200,8 @@ def should_fuse_mega_moe_shared_experts(moe: DeepseekV2MoE) -> bool:
|
||||
and is_sm100_supported()
|
||||
and _mega_moe_mma_type(moe.experts) == "fp8xfp4"
|
||||
and moe.shared_experts_is_fp8
|
||||
and moe.shared_experts_weight_block_size == [128, 128]
|
||||
and moe.shared_experts_weight_block_size is not None
|
||||
and moe.shared_experts_weight_block_size[1] % 32 == 0
|
||||
)
|
||||
|
||||
|
||||
@@ -617,10 +618,21 @@ def _transform_mega_moe_shared_sf(linear) -> torch.Tensor:
|
||||
if sf.dtype == torch.int32:
|
||||
# Requantized for the DeepGEMM linear runner.
|
||||
sf = inverse_transform_scale_ue8m0(sf, mn=n)[:, : k // 128]
|
||||
block_m, block_k = 128, 128
|
||||
else:
|
||||
block_m, block_k = linear.quant_method.quant_config.weight_block_size
|
||||
sf = sf.repeat_interleave(block_k // 32, dim=-1)
|
||||
if block_m == 128:
|
||||
return transform_sf_into_required_layout(
|
||||
sf, mn=n, k=k, recipe=(128, 32), disable_ue8m0_cast=False
|
||||
)
|
||||
if block_m != 1:
|
||||
sf = sf.repeat_interleave(block_m, dim=0)
|
||||
return transform_sf_into_required_layout(
|
||||
sf.repeat_interleave(4, dim=-1),
|
||||
sf.unsqueeze(0).contiguous(),
|
||||
mn=n,
|
||||
k=k,
|
||||
recipe=(128, 32),
|
||||
recipe=(1, 32),
|
||||
num_groups=1,
|
||||
disable_ue8m0_cast=False,
|
||||
)
|
||||
).squeeze(0)
|
||||
|
||||
Reference in New Issue
Block a user