[Kernel] Use Cake softmax for qualified SM103 sampling workloads (#42004)
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
# Cake kernels
|
||||
|
||||
This directory owns thin adapters to Cake-generated implementations distributed
|
||||
by FlashInfer. Generated CUDA sources and JIT caches remain owned by FlashInfer.
|
||||
As with `kda_kernels/`, runtime consumers import the operator facade under
|
||||
`sglang.kernels.ops`; importing the namespace must not initialize a backend.
|
||||
|
||||
Backend selection must preserve the operator's dtype, layout, state, mutation,
|
||||
and numerical contract. In particular, Cake top-k-then-top-p sampling is not a
|
||||
drop-in replacement for joint top-k/top-p filtering. Prepared plans must remain
|
||||
valid when input contents and buffers change across calls and CUDA Graph replay.
|
||||
|
||||
The sampler's softmax fast path accepts FP32 contiguous logits on SM103,
|
||||
small batches and large vocabularies. The sampler retains its existing filtering
|
||||
and random-number generation, and deterministic inference retains torch.softmax.
|
||||
Other operators require their own numerical and real-model performance evidence
|
||||
before being added here.
|
||||
|
||||
The fast path also requires FlashInfer's `cake_blackwell_softmax` module. Older
|
||||
FlashInfer installations retain Torch softmax; this adapter does not change
|
||||
SGLang's dependency pins.
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Lazy adapters for Cake-generated kernels distributed by FlashInfer.
|
||||
|
||||
Runtime callers use the corresponding ``sglang.kernels.ops`` facade. Importing
|
||||
this package does not load FlashInfer, initialize CUDA, or compile kernels.
|
||||
"""
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Blackwell Cake softmax through FlashInfer's public API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from importlib.util import find_spec
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _device_supported(device_index: int) -> bool:
|
||||
import torch
|
||||
|
||||
return (
|
||||
torch.cuda.get_device_capability(device_index) == (10, 3)
|
||||
and find_spec("flashinfer.jit.cake_blackwell_softmax") is not None
|
||||
)
|
||||
|
||||
|
||||
def supports_softmax(logits: torch.Tensor) -> bool:
|
||||
"""The large-vocabulary domain qualified against the eager Torch caller."""
|
||||
import torch
|
||||
|
||||
return (
|
||||
logits.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and logits.dtype == torch.float32
|
||||
and logits.ndim == 2
|
||||
and 1 <= logits.shape[0] <= 64
|
||||
and 128256 <= logits.shape[1] <= 262144
|
||||
and logits.shape[1] % 8 == 0
|
||||
and logits.is_contiguous()
|
||||
and _device_supported(logits.device.index)
|
||||
)
|
||||
|
||||
|
||||
def softmax(logits: torch.Tensor) -> torch.Tensor:
|
||||
"""Return softmax probabilities through FlashInfer's public API.
|
||||
|
||||
FlashInfer owns the architecture-specific Cake dispatch and its fallback
|
||||
for rows outside the optimized shape domain. No sampling distribution or
|
||||
random-number-generator behavior is changed by this operation.
|
||||
"""
|
||||
from flashinfer.sampling import softmax as flashinfer_softmax
|
||||
|
||||
return flashinfer_softmax(logits)
|
||||
@@ -6,11 +6,51 @@ from typing import TYPE_CHECKING, Union
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.selector import get_kernel
|
||||
from sglang.kernels.spec import FormatSignature, KernelBackend, KernelSpec
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="sampling.softmax",
|
||||
backend=KernelBackend.FLASHINFER,
|
||||
target="sglang.kernels.cake_kernels.sampling:softmax",
|
||||
capabilities=frozenset(
|
||||
{
|
||||
CapabilityRequirement.cuda(min_sm=(10, 3), max_sm=(10, 3)),
|
||||
}
|
||||
),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float32",),
|
||||
description="FP32 [batch, vocab] softmax probabilities",
|
||||
),
|
||||
description="Cake Blackwell softmax distributed by FlashInfer.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def cake_softmax(logits: torch.Tensor) -> torch.Tensor:
|
||||
"""Explicit Cake softmax entry point for Blackwell qualification."""
|
||||
return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits)
|
||||
|
||||
|
||||
def softmax(logits: torch.Tensor) -> torch.Tensor:
|
||||
"""Softmax for sampler logits, with a qualified Blackwell fast path."""
|
||||
import torch
|
||||
|
||||
from sglang.kernels.cake_kernels.sampling import supports_softmax
|
||||
|
||||
if supports_softmax(logits):
|
||||
return cake_softmax(logits)
|
||||
return torch.softmax(logits, dim=-1)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="sampling.top_k_renorm_probs",
|
||||
@@ -49,7 +89,7 @@ def top_p_renorm_probs(
|
||||
return get_kernel("sampling.top_p_renorm_probs", KernelBackend.AOT)(probs, top_p)
|
||||
|
||||
|
||||
__all__ = ["top_k_renorm_probs", "top_p_renorm_probs"]
|
||||
__all__ = ["cake_softmax", "softmax", "top_k_renorm_probs", "top_p_renorm_probs"]
|
||||
|
||||
|
||||
# Migrated from srt/layers/utils/hash.py (RFC #29630, Phase 2.5).
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax as sampler_softmax
|
||||
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -263,7 +264,11 @@ class Sampler(nn.Module):
|
||||
)
|
||||
|
||||
# In-place op to save memory
|
||||
logits[:] = torch.softmax(logits, dim=-1)
|
||||
logits[:] = (
|
||||
torch.softmax(logits, dim=-1)
|
||||
if self.enable_deterministic
|
||||
else sampler_softmax(logits)
|
||||
)
|
||||
probs = logits
|
||||
|
||||
batch_next_token_ids, sampling_mask_capture = self._sample_from_probs(
|
||||
|
||||
Reference in New Issue
Block a user