From 1a2eb6c426219312bfb000dfb36cf4afcfcf79b0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Fri, 2 Oct 2026 23:06:43 +0800 Subject: [PATCH] [Kernel] Use Cake softmax for qualified SM103 sampling workloads (#42004) --- python/sglang/kernels/cake_kernels/README.md | 21 ++++++++ .../sglang/kernels/cake_kernels/__init__.py | 5 ++ .../sglang/kernels/cake_kernels/sampling.py | 49 +++++++++++++++++++ .../sglang/kernels/ops/sampling/__init__.py | 44 ++++++++++++++++- python/sglang/srt/layers/sampler.py | 7 ++- 5 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 python/sglang/kernels/cake_kernels/README.md create mode 100644 python/sglang/kernels/cake_kernels/__init__.py create mode 100644 python/sglang/kernels/cake_kernels/sampling.py diff --git a/python/sglang/kernels/cake_kernels/README.md b/python/sglang/kernels/cake_kernels/README.md new file mode 100644 index 0000000000..ab59ace304 --- /dev/null +++ b/python/sglang/kernels/cake_kernels/README.md @@ -0,0 +1,21 @@ +# Cake kernels + +This directory owns thin adapters to Cake-generated implementations distributed +by FlashInfer. Generated CUDA sources and JIT caches remain owned by FlashInfer. +As with `kda_kernels/`, runtime consumers import the operator facade under +`sglang.kernels.ops`; importing the namespace must not initialize a backend. + +Backend selection must preserve the operator's dtype, layout, state, mutation, +and numerical contract. In particular, Cake top-k-then-top-p sampling is not a +drop-in replacement for joint top-k/top-p filtering. Prepared plans must remain +valid when input contents and buffers change across calls and CUDA Graph replay. + +The sampler's softmax fast path accepts FP32 contiguous logits on SM103, +small batches and large vocabularies. The sampler retains its existing filtering +and random-number generation, and deterministic inference retains torch.softmax. +Other operators require their own numerical and real-model performance evidence +before being added here. + +The fast path also requires FlashInfer's `cake_blackwell_softmax` module. Older +FlashInfer installations retain Torch softmax; this adapter does not change +SGLang's dependency pins. diff --git a/python/sglang/kernels/cake_kernels/__init__.py b/python/sglang/kernels/cake_kernels/__init__.py new file mode 100644 index 0000000000..56d05f4a93 --- /dev/null +++ b/python/sglang/kernels/cake_kernels/__init__.py @@ -0,0 +1,5 @@ +"""Lazy adapters for Cake-generated kernels distributed by FlashInfer. + +Runtime callers use the corresponding ``sglang.kernels.ops`` facade. Importing +this package does not load FlashInfer, initialize CUDA, or compile kernels. +""" diff --git a/python/sglang/kernels/cake_kernels/sampling.py b/python/sglang/kernels/cake_kernels/sampling.py new file mode 100644 index 0000000000..144dab9cda --- /dev/null +++ b/python/sglang/kernels/cake_kernels/sampling.py @@ -0,0 +1,49 @@ +"""Blackwell Cake softmax through FlashInfer's public API.""" + +from __future__ import annotations + +from functools import lru_cache +from importlib.util import find_spec +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import torch + + +@lru_cache(maxsize=None) +def _device_supported(device_index: int) -> bool: + import torch + + return ( + torch.cuda.get_device_capability(device_index) == (10, 3) + and find_spec("flashinfer.jit.cake_blackwell_softmax") is not None + ) + + +def supports_softmax(logits: torch.Tensor) -> bool: + """The large-vocabulary domain qualified against the eager Torch caller.""" + import torch + + return ( + logits.is_cuda + and torch.version.cuda is not None + and logits.dtype == torch.float32 + and logits.ndim == 2 + and 1 <= logits.shape[0] <= 64 + and 128256 <= logits.shape[1] <= 262144 + and logits.shape[1] % 8 == 0 + and logits.is_contiguous() + and _device_supported(logits.device.index) + ) + + +def softmax(logits: torch.Tensor) -> torch.Tensor: + """Return softmax probabilities through FlashInfer's public API. + + FlashInfer owns the architecture-specific Cake dispatch and its fallback + for rows outside the optimized shape domain. No sampling distribution or + random-number-generator behavior is changed by this operation. + """ + from flashinfer.sampling import softmax as flashinfer_softmax + + return flashinfer_softmax(logits) diff --git a/python/sglang/kernels/ops/sampling/__init__.py b/python/sglang/kernels/ops/sampling/__init__.py index 3e42c16ce6..81bd97becf 100644 --- a/python/sglang/kernels/ops/sampling/__init__.py +++ b/python/sglang/kernels/ops/sampling/__init__.py @@ -6,11 +6,51 @@ from typing import TYPE_CHECKING, Union from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel -from sglang.kernels.spec import FormatSignature, KernelBackend, KernelSpec +from sglang.kernels.spec import ( + CapabilityRequirement, + FormatSignature, + KernelBackend, + KernelSpec, +) if TYPE_CHECKING: import torch +register_kernel( + KernelSpec( + op="sampling.softmax", + backend=KernelBackend.FLASHINFER, + target="sglang.kernels.cake_kernels.sampling:softmax", + capabilities=frozenset( + { + CapabilityRequirement.cuda(min_sm=(10, 3), max_sm=(10, 3)), + } + ), + format_signature=FormatSignature( + supported_dtypes=("float32",), + description="FP32 [batch, vocab] softmax probabilities", + ), + description="Cake Blackwell softmax distributed by FlashInfer.", + ) +) + + +def cake_softmax(logits: torch.Tensor) -> torch.Tensor: + """Explicit Cake softmax entry point for Blackwell qualification.""" + return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits) + + +def softmax(logits: torch.Tensor) -> torch.Tensor: + """Softmax for sampler logits, with a qualified Blackwell fast path.""" + import torch + + from sglang.kernels.cake_kernels.sampling import supports_softmax + + if supports_softmax(logits): + return cake_softmax(logits) + return torch.softmax(logits, dim=-1) + + register_kernel( KernelSpec( op="sampling.top_k_renorm_probs", @@ -49,7 +89,7 @@ def top_p_renorm_probs( return get_kernel("sampling.top_p_renorm_probs", KernelBackend.AOT)(probs, top_p) -__all__ = ["top_k_renorm_probs", "top_p_renorm_probs"] +__all__ = ["cake_softmax", "softmax", "top_k_renorm_probs", "top_p_renorm_probs"] # Migrated from srt/layers/utils/hash.py (RFC #29630, Phase 2.5). diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index c1dae6ef24..c2b62fc68b 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -6,6 +6,7 @@ import torch import torch.distributed as dist from torch import nn +from sglang.kernels.ops.sampling import softmax as sampler_softmax from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32 from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import ( @@ -263,7 +264,11 @@ class Sampler(nn.Module): ) # In-place op to save memory - logits[:] = torch.softmax(logits, dim=-1) + logits[:] = ( + torch.softmax(logits, dim=-1) + if self.enable_deterministic + else sampler_softmax(logits) + ) probs = logits batch_next_token_ids, sampling_mask_capture = self._sample_from_probs(