[DSv4.1] mxfp8 dispatch cache (#42156)

Co-authored-by: Yuwei An <yuwei.an@MacBook-Pro-yuweian.local>
This commit is contained in:
Yuwei An
2026-10-09 16:36:28 -07:00
committed by GitHub
co-authored by Yuwei An
parent 46498bc8c5
commit 2d38934493
3 changed files with 260 additions and 1 deletions
@@ -493,6 +493,12 @@ if get_platform().is_blackwell and is_flashinfer_available():
from flashinfer import mxfp8_quantize as _raw_flashinfer_mxfp8_quantize
from flashinfer.gemm import gemm_fp8_nt_groupwise as _raw_gemm_fp8_nt_groupwise
from sglang.srt.layers.quantization.mxfp8_dispatch_cache import (
maybe_cache_mxfp8_dispatch,
)
_flashinfer_mm_mxfp8_impl = maybe_cache_mxfp8_dispatch(_raw_flashinfer_mm_mxfp8)
@lru_cache(maxsize=1)
def _get_flashinfer_groupwise_backend() -> str:
if get_fp8_gemm_runner_backend().is_flashinfer_cutlass():
@@ -587,7 +593,7 @@ if get_platform().is_blackwell and is_flashinfer_available():
use_8x4_sf_layout: bool = False,
backend: str = "auto",
) -> torch.Tensor:
return _raw_flashinfer_mm_mxfp8(
return _flashinfer_mm_mxfp8_impl(
q_input,
weight_t,
x_scale_u8,
@@ -0,0 +1,89 @@
"""Cache compatible FlashInfer 0.7.0+ MXFP8 host dispatch.
Keep FlashInfer's output allocation, autotuner and kernel invocation intact.
Only reuse the stateless CuTe runner and validation of identical metadata.
"""
from functools import lru_cache
from inspect import Parameter, signature
from packaging.version import InvalidVersion, Version
try:
from flashinfer import __version__ as flashinfer_version
from flashinfer.gemm import gemm_base
except ImportError:
flashinfer_version = ""
gemm_base = None
_cached_runner_factories = set()
def maybe_cache_mxfp8_dispatch(raw_mm):
"""Keep unsupported FlashInfer versions and APIs on the original path."""
if gemm_base is None:
return raw_mm
try:
if Version(flashinfer_version) < Version("0.7.0"):
return raw_mm
except InvalidVersion:
return raw_mm
try:
if not callable(gemm_base._cute_dsl_gemm_mxfp8_runner):
return raw_mm
parameters = signature(raw_mm, follow_wrapped=False).parameters
except (AttributeError, TypeError, ValueError):
return raw_mm
skip_check = parameters.get("skip_check")
accepts_keyword = skip_check is not None and skip_check.kind in (
Parameter.POSITIONAL_OR_KEYWORD,
Parameter.KEYWORD_ONLY,
)
if not accepts_keyword and not any(
p.kind == Parameter.VAR_KEYWORD for p in parameters.values()
):
return raw_mm
return Mxfp8DispatchCache(raw_mm, gemm_base)
class Mxfp8DispatchCache:
def __init__(self, raw_mm, gemm_module):
self.raw_mm = raw_mm
self.validated_set = set()
factory = gemm_module._cute_dsl_gemm_mxfp8_runner
# This process-wide patch only memoizes the stateless
# runner factory. The factory arguments include SM, PDL and output dtype.
if factory not in _cached_runner_factories:
factory = lru_cache(maxsize=16)(factory)
gemm_module._cute_dsl_gemm_mxfp8_runner = factory
_cached_runner_factories.add(factory)
self.runner_factory = factory
def __call__(self, a, b, a_scale, b_scale, out_dtype, use_8x4_sf_layout, backend):
kwargs = dict(
out_dtype=out_dtype,
use_8x4_sf_layout=use_8x4_sf_layout,
backend=backend,
)
if backend != "cute-dsl":
return self.raw_mm(a, b, a_scale, b_scale, **kwargs)
# Validation depends on metadata, never tensor contents. A changed
# shape, stride, dtype or device must pass the original checks again.
key = (
out_dtype,
use_8x4_sf_layout,
tuple(
(t.shape, t.stride(), t.dtype, t.device, t.layout)
for t in (a, b, a_scale, b_scale)
),
)
is_validated = key in self.validated_set
result = self.raw_mm(a, b, a_scale, b_scale, skip_check=is_validated, **kwargs)
if not is_validated:
# Bound metadata memory for varying workloads. Only successful
# executions qualify; a failed validation is never cached.
if len(self.validated_set) >= 1024:
self.validated_set.clear()
self.validated_set.add(key)
return result