[Kernel] Use Cake softmax for qualified SM103 sampling workloads (#42004)

This commit is contained in:
Xiaoyu Zhang
2026-10-02 23:06:43 +08:00
committed by GitHub
parent bb9a820f09
commit 1a2eb6c426
5 changed files with 123 additions and 3 deletions
@@ -0,0 +1,21 @@
# Cake kernels
This directory owns thin adapters to Cake-generated implementations distributed
by FlashInfer. Generated CUDA sources and JIT caches remain owned by FlashInfer.
As with `kda_kernels/`, runtime consumers import the operator facade under
`sglang.kernels.ops`; importing the namespace must not initialize a backend.
Backend selection must preserve the operator's dtype, layout, state, mutation,
and numerical contract. In particular, Cake top-k-then-top-p sampling is not a
drop-in replacement for joint top-k/top-p filtering. Prepared plans must remain
valid when input contents and buffers change across calls and CUDA Graph replay.
The sampler's softmax fast path accepts FP32 contiguous logits on SM103,
small batches and large vocabularies. The sampler retains its existing filtering
and random-number generation, and deterministic inference retains torch.softmax.
Other operators require their own numerical and real-model performance evidence
before being added here.
The fast path also requires FlashInfer's `cake_blackwell_softmax` module. Older
FlashInfer installations retain Torch softmax; this adapter does not change
SGLang's dependency pins.
@@ -0,0 +1,5 @@
"""Lazy adapters for Cake-generated kernels distributed by FlashInfer.
Runtime callers use the corresponding ``sglang.kernels.ops`` facade. Importing
this package does not load FlashInfer, initialize CUDA, or compile kernels.
"""
@@ -0,0 +1,49 @@
"""Blackwell Cake softmax through FlashInfer's public API."""
from __future__ import annotations
from functools import lru_cache
from importlib.util import find_spec
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import torch
@lru_cache(maxsize=None)
def _device_supported(device_index: int) -> bool:
import torch
return (
torch.cuda.get_device_capability(device_index) == (10, 3)
and find_spec("flashinfer.jit.cake_blackwell_softmax") is not None
)
def supports_softmax(logits: torch.Tensor) -> bool:
"""The large-vocabulary domain qualified against the eager Torch caller."""
import torch
return (
logits.is_cuda
and torch.version.cuda is not None
and logits.dtype == torch.float32
and logits.ndim == 2
and 1 <= logits.shape[0] <= 64
and 128256 <= logits.shape[1] <= 262144
and logits.shape[1] % 8 == 0
and logits.is_contiguous()
and _device_supported(logits.device.index)
)
def softmax(logits: torch.Tensor) -> torch.Tensor:
"""Return softmax probabilities through FlashInfer's public API.
FlashInfer owns the architecture-specific Cake dispatch and its fallback
for rows outside the optimized shape domain. No sampling distribution or
random-number-generator behavior is changed by this operation.
"""
from flashinfer.sampling import softmax as flashinfer_softmax
return flashinfer_softmax(logits)
+42 -2
View File
@@ -6,11 +6,51 @@ from typing import TYPE_CHECKING, Union
from sglang.kernels.registry import register_kernel
from sglang.kernels.selector import get_kernel
from sglang.kernels.spec import FormatSignature, KernelBackend, KernelSpec
from sglang.kernels.spec import (
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
)
if TYPE_CHECKING:
import torch
register_kernel(
KernelSpec(
op="sampling.softmax",
backend=KernelBackend.FLASHINFER,
target="sglang.kernels.cake_kernels.sampling:softmax",
capabilities=frozenset(
{
CapabilityRequirement.cuda(min_sm=(10, 3), max_sm=(10, 3)),
}
),
format_signature=FormatSignature(
supported_dtypes=("float32",),
description="FP32 [batch, vocab] softmax probabilities",
),
description="Cake Blackwell softmax distributed by FlashInfer.",
)
)
def cake_softmax(logits: torch.Tensor) -> torch.Tensor:
"""Explicit Cake softmax entry point for Blackwell qualification."""
return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits)
def softmax(logits: torch.Tensor) -> torch.Tensor:
"""Softmax for sampler logits, with a qualified Blackwell fast path."""
import torch
from sglang.kernels.cake_kernels.sampling import supports_softmax
if supports_softmax(logits):
return cake_softmax(logits)
return torch.softmax(logits, dim=-1)
register_kernel(
KernelSpec(
op="sampling.top_k_renorm_probs",
@@ -49,7 +89,7 @@ def top_p_renorm_probs(
return get_kernel("sampling.top_p_renorm_probs", KernelBackend.AOT)(probs, top_p)
__all__ = ["top_k_renorm_probs", "top_p_renorm_probs"]
__all__ = ["cake_softmax", "softmax", "top_k_renorm_probs", "top_p_renorm_probs"]
# Migrated from srt/layers/utils/hash.py (RFC #29630, Phase 2.5).
+6 -1
View File
@@ -6,6 +6,7 @@ import torch
import torch.distributed as dist
from torch import nn
from sglang.kernels.ops.sampling import softmax as sampler_softmax
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
@@ -263,7 +264,11 @@ class Sampler(nn.Module):
)
# In-place op to save memory
logits[:] = torch.softmax(logits, dim=-1)
logits[:] = (
torch.softmax(logits, dim=-1)
if self.enable_deterministic
else sampler_softmax(logits)
)
probs = logits
batch_next_token_ids, sampling_mask_capture = self._sample_from_probs(