From f05ad5ec0139d1a41e04115384f6f03b0ac4d19f Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 23 Oct 2026 15:06:09 -0700 Subject: [PATCH] [Perf] logits: Cast bf16 logits into the fp32 buffer with a Triton kernel (#46167) --- python/sglang/kernels/ops/activation/__init__.py | 8 ++++++++ python/sglang/kernels/ops/activation/softcap.py | 14 +++++++++++--- python/sglang/srt/layers/logits_processor.py | 4 ++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/python/sglang/kernels/ops/activation/__init__.py b/python/sglang/kernels/ops/activation/__init__.py index 2f81a27a73..1ce8c59974 100644 --- a/python/sglang/kernels/ops/activation/__init__.py +++ b/python/sglang/kernels/ops/activation/__init__.py @@ -143,6 +143,14 @@ register_kernel( capabilities=CUDA_ONLY, ) ) +register_kernel( + KernelSpec( + op="activation.cast_to_float32_logits", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.activation.softcap:cast_to_float32_logits", + capabilities=CUDA_ONLY, + ) +) # Kernels introduced with Kimi-K3, inventoried by logical operator group. diff --git a/python/sglang/kernels/ops/activation/softcap.py b/python/sglang/kernels/ops/activation/softcap.py index 46424ca363..d1025c4510 100644 --- a/python/sglang/kernels/ops/activation/softcap.py +++ b/python/sglang/kernels/ops/activation/softcap.py @@ -14,6 +14,7 @@ def softcap_inplace_logits_kernel( output_ptr=None, output_row_stride=0, CAST_TO_FP32: tl.constexpr = False, + APPLY_SOFTCAP: tl.constexpr = True, ): row = tl.program_id(1).to(tl.int64) pid = tl.program_id(0).to(tl.int64) @@ -27,9 +28,10 @@ def softcap_inplace_logits_kernel( if CAST_TO_FP32: x = x.to(tl.float32) - x = x / softcapping_value - x = libdevice.tanh(x) - x = x * softcapping_value + if APPLY_SOFTCAP: + x = x / softcapping_value + x = libdevice.tanh(x) + x = x * softcapping_value # Store result if CAST_TO_FP32: @@ -70,6 +72,7 @@ def _softcap_logits(full_logits, final_logit_softcapping, output=None): if output is not None else 0, CAST_TO_FP32=output is not None, + APPLY_SOFTCAP=final_logit_softcapping is not None, ) return full_logits if output is None else output @@ -83,6 +86,11 @@ def softcap_to_float32_logits(logits, final_logit_softcapping, output): return _softcap_logits(logits, final_logit_softcapping, output) +def cast_to_float32_logits(logits, output): + """Write FP32 logits to an equally shaped, row-contiguous output.""" + return _softcap_logits(logits, None, output) + + @triton.jit def _gathered_softcap_logits_kernel( gathered_ptr, diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 2122b89b8b..2d16b98cd4 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -24,6 +24,7 @@ import torch from torch import nn from sglang.kernels.ops.activation.softcap import ( + cast_to_float32_logits, gathered_softcap_logits, ) from sglang.kernels.ops.activation.softcap import ( @@ -1279,6 +1280,9 @@ class LogitsProcessor(nn.Module): logits.shape ): assert logits_buffer.dtype == torch.float + if logits.is_cuda and logits.dtype in (torch.float16, torch.bfloat16): + # Measured on GB300: torch's dtype-casting copy_ is ~2.8x slower. + return cast_to_float32_logits(logits, logits_buffer) logits_buffer.copy_(logits) logits = logits_buffer else: