[DSv4.1] mxfp8 dispatch cache (#42156)
Co-authored-by: Yuwei An <yuwei.an@MacBook-Pro-yuweian.local>
This commit is contained in:
@@ -493,6 +493,12 @@ if get_platform().is_blackwell and is_flashinfer_available():
|
||||
from flashinfer import mxfp8_quantize as _raw_flashinfer_mxfp8_quantize
|
||||
from flashinfer.gemm import gemm_fp8_nt_groupwise as _raw_gemm_fp8_nt_groupwise
|
||||
|
||||
from sglang.srt.layers.quantization.mxfp8_dispatch_cache import (
|
||||
maybe_cache_mxfp8_dispatch,
|
||||
)
|
||||
|
||||
_flashinfer_mm_mxfp8_impl = maybe_cache_mxfp8_dispatch(_raw_flashinfer_mm_mxfp8)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_flashinfer_groupwise_backend() -> str:
|
||||
if get_fp8_gemm_runner_backend().is_flashinfer_cutlass():
|
||||
@@ -587,7 +593,7 @@ if get_platform().is_blackwell and is_flashinfer_available():
|
||||
use_8x4_sf_layout: bool = False,
|
||||
backend: str = "auto",
|
||||
) -> torch.Tensor:
|
||||
return _raw_flashinfer_mm_mxfp8(
|
||||
return _flashinfer_mm_mxfp8_impl(
|
||||
q_input,
|
||||
weight_t,
|
||||
x_scale_u8,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Cache compatible FlashInfer 0.7.0+ MXFP8 host dispatch.
|
||||
|
||||
Keep FlashInfer's output allocation, autotuner and kernel invocation intact.
|
||||
Only reuse the stateless CuTe runner and validation of identical metadata.
|
||||
"""
|
||||
|
||||
from functools import lru_cache
|
||||
from inspect import Parameter, signature
|
||||
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
try:
|
||||
from flashinfer import __version__ as flashinfer_version
|
||||
from flashinfer.gemm import gemm_base
|
||||
except ImportError:
|
||||
flashinfer_version = ""
|
||||
gemm_base = None
|
||||
|
||||
_cached_runner_factories = set()
|
||||
|
||||
|
||||
def maybe_cache_mxfp8_dispatch(raw_mm):
|
||||
"""Keep unsupported FlashInfer versions and APIs on the original path."""
|
||||
if gemm_base is None:
|
||||
return raw_mm
|
||||
try:
|
||||
if Version(flashinfer_version) < Version("0.7.0"):
|
||||
return raw_mm
|
||||
except InvalidVersion:
|
||||
return raw_mm
|
||||
try:
|
||||
if not callable(gemm_base._cute_dsl_gemm_mxfp8_runner):
|
||||
return raw_mm
|
||||
parameters = signature(raw_mm, follow_wrapped=False).parameters
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return raw_mm
|
||||
skip_check = parameters.get("skip_check")
|
||||
accepts_keyword = skip_check is not None and skip_check.kind in (
|
||||
Parameter.POSITIONAL_OR_KEYWORD,
|
||||
Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
if not accepts_keyword and not any(
|
||||
p.kind == Parameter.VAR_KEYWORD for p in parameters.values()
|
||||
):
|
||||
return raw_mm
|
||||
return Mxfp8DispatchCache(raw_mm, gemm_base)
|
||||
|
||||
|
||||
class Mxfp8DispatchCache:
|
||||
def __init__(self, raw_mm, gemm_module):
|
||||
self.raw_mm = raw_mm
|
||||
self.validated_set = set()
|
||||
factory = gemm_module._cute_dsl_gemm_mxfp8_runner
|
||||
# This process-wide patch only memoizes the stateless
|
||||
# runner factory. The factory arguments include SM, PDL and output dtype.
|
||||
if factory not in _cached_runner_factories:
|
||||
factory = lru_cache(maxsize=16)(factory)
|
||||
gemm_module._cute_dsl_gemm_mxfp8_runner = factory
|
||||
_cached_runner_factories.add(factory)
|
||||
self.runner_factory = factory
|
||||
|
||||
def __call__(self, a, b, a_scale, b_scale, out_dtype, use_8x4_sf_layout, backend):
|
||||
kwargs = dict(
|
||||
out_dtype=out_dtype,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
backend=backend,
|
||||
)
|
||||
if backend != "cute-dsl":
|
||||
return self.raw_mm(a, b, a_scale, b_scale, **kwargs)
|
||||
|
||||
# Validation depends on metadata, never tensor contents. A changed
|
||||
# shape, stride, dtype or device must pass the original checks again.
|
||||
key = (
|
||||
out_dtype,
|
||||
use_8x4_sf_layout,
|
||||
tuple(
|
||||
(t.shape, t.stride(), t.dtype, t.device, t.layout)
|
||||
for t in (a, b, a_scale, b_scale)
|
||||
),
|
||||
)
|
||||
is_validated = key in self.validated_set
|
||||
result = self.raw_mm(a, b, a_scale, b_scale, skip_check=is_validated, **kwargs)
|
||||
if not is_validated:
|
||||
# Bound metadata memory for varying workloads. Only successful
|
||||
# executions qualify; a failed validation is never cached.
|
||||
if len(self.validated_set) >= 1024:
|
||||
self.validated_set.clear()
|
||||
self.validated_set.add(key)
|
||||
return result
|
||||
Reference in New Issue
Block a user