From 2d3893449380864c178ec74be0a6bf24d0131175 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Fri, 9 Oct 2026 16:36:28 -0700 Subject: [PATCH] [DSv4.1] mxfp8 dispatch cache (#42156) Co-authored-by: Yuwei An --- .../srt/layers/quantization/fp8_utils.py | 8 +- .../quantization/mxfp8_dispatch_cache.py | 89 ++++++++++ .../quantization/test_mxfp8_dispatch_cache.py | 164 ++++++++++++++++++ 3 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 python/sglang/srt/layers/quantization/mxfp8_dispatch_cache.py create mode 100644 test/manual/quantization/test_mxfp8_dispatch_cache.py diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 867666bec1..2274f3c0c9 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -493,6 +493,12 @@ if get_platform().is_blackwell and is_flashinfer_available(): from flashinfer import mxfp8_quantize as _raw_flashinfer_mxfp8_quantize from flashinfer.gemm import gemm_fp8_nt_groupwise as _raw_gemm_fp8_nt_groupwise + from sglang.srt.layers.quantization.mxfp8_dispatch_cache import ( + maybe_cache_mxfp8_dispatch, + ) + + _flashinfer_mm_mxfp8_impl = maybe_cache_mxfp8_dispatch(_raw_flashinfer_mm_mxfp8) + @lru_cache(maxsize=1) def _get_flashinfer_groupwise_backend() -> str: if get_fp8_gemm_runner_backend().is_flashinfer_cutlass(): @@ -587,7 +593,7 @@ if get_platform().is_blackwell and is_flashinfer_available(): use_8x4_sf_layout: bool = False, backend: str = "auto", ) -> torch.Tensor: - return _raw_flashinfer_mm_mxfp8( + return _flashinfer_mm_mxfp8_impl( q_input, weight_t, x_scale_u8, diff --git a/python/sglang/srt/layers/quantization/mxfp8_dispatch_cache.py b/python/sglang/srt/layers/quantization/mxfp8_dispatch_cache.py new file mode 100644 index 0000000000..2a47339625 --- /dev/null +++ b/python/sglang/srt/layers/quantization/mxfp8_dispatch_cache.py @@ -0,0 +1,89 @@ +"""Cache compatible FlashInfer 0.7.0+ MXFP8 host dispatch. + +Keep FlashInfer's output allocation, autotuner and kernel invocation intact. +Only reuse the stateless CuTe runner and validation of identical metadata. +""" + +from functools import lru_cache +from inspect import Parameter, signature + +from packaging.version import InvalidVersion, Version + +try: + from flashinfer import __version__ as flashinfer_version + from flashinfer.gemm import gemm_base +except ImportError: + flashinfer_version = "" + gemm_base = None + +_cached_runner_factories = set() + + +def maybe_cache_mxfp8_dispatch(raw_mm): + """Keep unsupported FlashInfer versions and APIs on the original path.""" + if gemm_base is None: + return raw_mm + try: + if Version(flashinfer_version) < Version("0.7.0"): + return raw_mm + except InvalidVersion: + return raw_mm + try: + if not callable(gemm_base._cute_dsl_gemm_mxfp8_runner): + return raw_mm + parameters = signature(raw_mm, follow_wrapped=False).parameters + except (AttributeError, TypeError, ValueError): + return raw_mm + skip_check = parameters.get("skip_check") + accepts_keyword = skip_check is not None and skip_check.kind in ( + Parameter.POSITIONAL_OR_KEYWORD, + Parameter.KEYWORD_ONLY, + ) + if not accepts_keyword and not any( + p.kind == Parameter.VAR_KEYWORD for p in parameters.values() + ): + return raw_mm + return Mxfp8DispatchCache(raw_mm, gemm_base) + + +class Mxfp8DispatchCache: + def __init__(self, raw_mm, gemm_module): + self.raw_mm = raw_mm + self.validated_set = set() + factory = gemm_module._cute_dsl_gemm_mxfp8_runner + # This process-wide patch only memoizes the stateless + # runner factory. The factory arguments include SM, PDL and output dtype. + if factory not in _cached_runner_factories: + factory = lru_cache(maxsize=16)(factory) + gemm_module._cute_dsl_gemm_mxfp8_runner = factory + _cached_runner_factories.add(factory) + self.runner_factory = factory + + def __call__(self, a, b, a_scale, b_scale, out_dtype, use_8x4_sf_layout, backend): + kwargs = dict( + out_dtype=out_dtype, + use_8x4_sf_layout=use_8x4_sf_layout, + backend=backend, + ) + if backend != "cute-dsl": + return self.raw_mm(a, b, a_scale, b_scale, **kwargs) + + # Validation depends on metadata, never tensor contents. A changed + # shape, stride, dtype or device must pass the original checks again. + key = ( + out_dtype, + use_8x4_sf_layout, + tuple( + (t.shape, t.stride(), t.dtype, t.device, t.layout) + for t in (a, b, a_scale, b_scale) + ), + ) + is_validated = key in self.validated_set + result = self.raw_mm(a, b, a_scale, b_scale, skip_check=is_validated, **kwargs) + if not is_validated: + # Bound metadata memory for varying workloads. Only successful + # executions qualify; a failed validation is never cached. + if len(self.validated_set) >= 1024: + self.validated_set.clear() + self.validated_set.add(key) + return result diff --git a/test/manual/quantization/test_mxfp8_dispatch_cache.py b/test/manual/quantization/test_mxfp8_dispatch_cache.py new file mode 100644 index 0000000000..853b1c207a --- /dev/null +++ b/test/manual/quantization/test_mxfp8_dispatch_cache.py @@ -0,0 +1,164 @@ +"""CPU-only safety checks for the metadata-validation cache.""" + +import importlib.util +import unittest +from functools import wraps +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +_path = ( + Path(__file__).resolve().parents[3] + / "python/sglang/srt/layers/quantization/mxfp8_dispatch_cache.py" +) +_spec = importlib.util.spec_from_file_location("mxfp8_dispatch_cache", _path) +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) + + +class TensorMetadata: + def __init__(self, dtype="fp8", shape=(8, 128), strides=(128, 1)): + self.dtype = dtype + self.shape = shape + self.strides = strides + self.device = "cuda:0" + self.layout = "strided" + + def stride(self): + return self.strides + + +class DispatchCacheTest(unittest.TestCase): + def setUp(self): + self.checks = [] + + def raw(a, b, a_scale, b_scale, skip_check=False, **kwargs): + self.checks.append(skip_check) + if not skip_check and a.dtype != "fp8": + raise ValueError("invalid dtype") + return object() + + self.cache = _module.Mxfp8DispatchCache( + raw, SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=lambda *args: object()) + ) + self.args = [TensorMetadata() for _ in range(4)] + self.kwargs = dict( + out_dtype="bf16", use_8x4_sf_layout=False, backend="cute-dsl" + ) + + def test_reuses_validation_but_not_outputs(self): + first = self.cache(*self.args, **self.kwargs) + second = self.cache(*[TensorMetadata() for _ in range(4)], **self.kwargs) + self.assertIsNot(first, second) + self.assertEqual(self.checks, [False, True]) + + def test_runner_factory_is_wrapped_once_and_reuses_matching_arguments(self): + module = SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=lambda *args: object()) + first = _module.Mxfp8DispatchCache(self.cache.raw_mm, module) + second = _module.Mxfp8DispatchCache(self.cache.raw_mm, module) + self.assertIs(first.runner_factory, second.runner_factory) + self.assertIs(module._cute_dsl_gemm_mxfp8_runner, first.runner_factory) + runner = first.runner_factory(10, 0, True, "bf16") + self.assertIs(runner, second.runner_factory(10, 0, True, "bf16")) + self.assertIsNot(runner, second.runner_factory(10, 0, False, "bf16")) + + def test_dtype_change_is_revalidated_and_failed_check_not_cached(self): + self.cache(*self.args, **self.kwargs) + self.args[0].dtype = "bf16" + for _ in range(2): + with self.assertRaisesRegex(ValueError, "invalid dtype"): + self.cache(*self.args, **self.kwargs) + self.assertEqual(self.checks, [False, False, False]) + + def test_shape_stride_and_device_changes_are_revalidated(self): + self.cache(*self.args, **self.kwargs) + for attribute, value in ( + ("shape", (16, 128)), + ("strides", (256, 1)), + ("device", "cuda:1"), + ): + setattr(self.args[0], attribute, value) + self.cache(*self.args, **self.kwargs) + self.assertEqual(self.checks, [False] * 4) + + def test_other_backends_keep_original_validation(self): + self.kwargs["backend"] = "auto" + for _ in range(2): + self.cache(*self.args, **self.kwargs) + self.assertEqual(self.checks, [False, False]) + + def test_supported_flashinfer_api_enables_cache(self): + for version in ( + "0.7.0", + "0.7.0+cu130", + "0.7.0.post1", + "0.7.0.post1+cu130", + "0.7.1", + "0.7.01", + "0.8.0", + "0.10.0", + "1.0.0", + ): + with ( + self.subTest(version=version), + patch.multiple( + _module, + flashinfer_version=version, + gemm_base=SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=lambda: None), + ), + ): + cached = _module.maybe_cache_mxfp8_dispatch(self.cache.raw_mm) + self.assertIsInstance(cached, _module.Mxfp8DispatchCache) + + def test_decorated_validation_api_accepts_skip_check(self): + def public_api(a, b, a_scale, b_scale): + pass + + @wraps(public_api) + def decorated(*args, **kwargs): + return self.cache.raw_mm(*args, **kwargs) + + with patch.multiple( + _module, + flashinfer_version="0.7.0", + gemm_base=SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=lambda: None), + ): + cached = _module.maybe_cache_mxfp8_dispatch(decorated) + self.assertIsInstance(cached, _module.Mxfp8DispatchCache) + cached(*self.args, **self.kwargs) + cached(*self.args, **self.kwargs) + self.assertEqual(self.checks, [False, True]) + + def test_unsupported_version_does_not_modify_runner(self): + for version in ("0.6.18", "0.7.0rc1", "0.7.0.dev1", "", "unknown"): + with self.subTest(version=version): + factory = lambda: None + module = SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=factory) + raw = self.cache.raw_mm + with patch.multiple( + _module, flashinfer_version=version, gemm_base=module + ): + self.assertIs(_module.maybe_cache_mxfp8_dispatch(raw), raw) + self.assertIs(module._cute_dsl_gemm_mxfp8_runner, factory) + + def test_missing_private_api_keeps_original_dispatch(self): + raw = self.cache.raw_mm + for module in ( + None, + SimpleNamespace(), + SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=None), + ): + with patch.multiple(_module, flashinfer_version="0.7.0", gemm_base=module): + self.assertIs(_module.maybe_cache_mxfp8_dispatch(raw), raw) + unsupported_raw = lambda a, b: None + factory = lambda: None + module = SimpleNamespace(_cute_dsl_gemm_mxfp8_runner=factory) + with patch.multiple(_module, flashinfer_version="0.7.0", gemm_base=module): + self.assertIs( + _module.maybe_cache_mxfp8_dispatch(unsupported_raw), unsupported_raw + ) + self.assertIs(module._cute_dsl_gemm_mxfp8_runner, factory) + + +if __name__ == "__main__": + unittest.main()