[Perf] dsv41: Use tuned MXFP8 tactics at decode sizes (#45630)

This commit is contained in:
Khoa Pham
2026-10-21 13:35:56 -07:00
committed by GitHub
parent a2e4ffbcea
commit 028de2c5d4
2 changed files with 3 additions and 33 deletions
+2 -2
View File
@@ -4578,8 +4578,8 @@ class DeepseekV4ForCausalLM(nn.Module):
if backend is None or not backend.is_flashinfer_cutedsl():
continue
if method.block_fp8_as_mxfp8:
# Small shapes and deterministic execution keep their pinned tactic.
method.mxfp8_prefill_autotune_min_tokens = 4096
# Deterministic execution keeps its pinned tactic.
method.mxfp8_prefill_autotune_min_tokens = 1
weight = layer.weight
scale = layer.weight_scale_inv_swizzled
key = (
@@ -487,7 +487,7 @@ class TestBlockFp8AsMxfp8PrefillAutotune(_LinearBackendCheck):
self.assertEqual(methods[0].apply.call_args.args[1].shape, (4096, 128))
methods[1].apply.assert_not_called()
for method in methods:
self.assertEqual(method.mxfp8_prefill_autotune_min_tokens, 4096)
self.assertEqual(method.mxfp8_prefill_autotune_min_tokens, 1)
self.assertIsNone(fallback.quant_method.mxfp8_prefill_autotune_min_tokens)
def test_block_fp8_dispatch_keeps_decode_and_determinism_pinned(self):
@@ -522,36 +522,6 @@ class TestBlockFp8AsMxfp8PrefillAutotune(_LinearBackendCheck):
method.apply(layer, torch.empty(rows, 128, device="cuda"))
self.assertEqual(call.call_args.kwargs.get("pin_tactic"), expected)
def test_prefill_tuning_leaves_decode_bit_identical(self):
"""Tuning the prefill buckets must not move the decode tactic: below the
stamped min_tokens the output has to stay bit-for-bit what it was."""
from flashinfer.autotuner import autotune
from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM
runtime_patch = mock.patch(
"sglang.srt.runtime_context.get_exec",
return_value=SimpleNamespace(
deterministic=SimpleNamespace(enable_deterministic_inference=False)
),
)
runtime_patch.start()
self.addCleanup(runtime_patch.stop)
layer = self._ready_layer(1792, 5120)
method = layer.quant_method
x = torch.randn(6, 5120, device="cuda", dtype=torch.bfloat16)
original = method.apply(layer, x)
model = SimpleNamespace(
config=SimpleNamespace(model_type="deepseek_v41"),
model=torch.nn.ModuleList([layer]),
)
with autotune(True):
DeepseekV4ForCausalLM.autotune_prefill_kernels(
model, 4096, dtype=torch.bfloat16
)
self.assertEqual(method.mxfp8_prefill_autotune_min_tokens, 4096)
torch.testing.assert_close(method.apply(layer, x), original, rtol=0, atol=0)
@unittest.skipIf(get_device_sm() < 90, "FP8 GEMM backends require SM90+")
class TestModeloptFp8PerTensorLinear(_LinearBackendCheck):