[Perf] logits: Cast bf16 logits into the fp32 buffer with a Triton kernel (#46167)

This commit is contained in:
Liangsheng Yin
2026-10-23 15:06:09 -07:00
committed by GitHub
parent 15dc838f9b
commit f05ad5ec01
3 changed files with 23 additions and 3 deletions
@@ -143,6 +143,14 @@ register_kernel(
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="activation.cast_to_float32_logits",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.activation.softcap:cast_to_float32_logits",
capabilities=CUDA_ONLY,
)
)
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
@@ -14,6 +14,7 @@ def softcap_inplace_logits_kernel(
output_ptr=None,
output_row_stride=0,
CAST_TO_FP32: tl.constexpr = False,
APPLY_SOFTCAP: tl.constexpr = True,
):
row = tl.program_id(1).to(tl.int64)
pid = tl.program_id(0).to(tl.int64)
@@ -27,9 +28,10 @@ def softcap_inplace_logits_kernel(
if CAST_TO_FP32:
x = x.to(tl.float32)
x = x / softcapping_value
x = libdevice.tanh(x)
x = x * softcapping_value
if APPLY_SOFTCAP:
x = x / softcapping_value
x = libdevice.tanh(x)
x = x * softcapping_value
# Store result
if CAST_TO_FP32:
@@ -70,6 +72,7 @@ def _softcap_logits(full_logits, final_logit_softcapping, output=None):
if output is not None
else 0,
CAST_TO_FP32=output is not None,
APPLY_SOFTCAP=final_logit_softcapping is not None,
)
return full_logits if output is None else output
@@ -83,6 +86,11 @@ def softcap_to_float32_logits(logits, final_logit_softcapping, output):
return _softcap_logits(logits, final_logit_softcapping, output)
def cast_to_float32_logits(logits, output):
"""Write FP32 logits to an equally shaped, row-contiguous output."""
return _softcap_logits(logits, None, output)
@triton.jit
def _gathered_softcap_logits_kernel(
gathered_ptr,
@@ -24,6 +24,7 @@ import torch
from torch import nn
from sglang.kernels.ops.activation.softcap import (
cast_to_float32_logits,
gathered_softcap_logits,
)
from sglang.kernels.ops.activation.softcap import (
@@ -1279,6 +1280,9 @@ class LogitsProcessor(nn.Module):
logits.shape
):
assert logits_buffer.dtype == torch.float
if logits.is_cuda and logits.dtype in (torch.float16, torch.bfloat16):
# Measured on GB300: torch's dtype-casting copy_ is ~2.8x slower.
return cast_to_float32_logits(logits, logits_buffer)
logits_buffer.copy_(logits)
logits = logits_buffer
else: