[Perf] logits: Cast bf16 logits into the fp32 buffer with a Triton kernel (#46167)
This commit is contained in:
@@ -143,6 +143,14 @@ register_kernel(
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="activation.cast_to_float32_logits",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.activation.softcap:cast_to_float32_logits",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
|
||||
@@ -14,6 +14,7 @@ def softcap_inplace_logits_kernel(
|
||||
output_ptr=None,
|
||||
output_row_stride=0,
|
||||
CAST_TO_FP32: tl.constexpr = False,
|
||||
APPLY_SOFTCAP: tl.constexpr = True,
|
||||
):
|
||||
row = tl.program_id(1).to(tl.int64)
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
@@ -27,9 +28,10 @@ def softcap_inplace_logits_kernel(
|
||||
if CAST_TO_FP32:
|
||||
x = x.to(tl.float32)
|
||||
|
||||
x = x / softcapping_value
|
||||
x = libdevice.tanh(x)
|
||||
x = x * softcapping_value
|
||||
if APPLY_SOFTCAP:
|
||||
x = x / softcapping_value
|
||||
x = libdevice.tanh(x)
|
||||
x = x * softcapping_value
|
||||
|
||||
# Store result
|
||||
if CAST_TO_FP32:
|
||||
@@ -70,6 +72,7 @@ def _softcap_logits(full_logits, final_logit_softcapping, output=None):
|
||||
if output is not None
|
||||
else 0,
|
||||
CAST_TO_FP32=output is not None,
|
||||
APPLY_SOFTCAP=final_logit_softcapping is not None,
|
||||
)
|
||||
return full_logits if output is None else output
|
||||
|
||||
@@ -83,6 +86,11 @@ def softcap_to_float32_logits(logits, final_logit_softcapping, output):
|
||||
return _softcap_logits(logits, final_logit_softcapping, output)
|
||||
|
||||
|
||||
def cast_to_float32_logits(logits, output):
|
||||
"""Write FP32 logits to an equally shaped, row-contiguous output."""
|
||||
return _softcap_logits(logits, None, output)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gathered_softcap_logits_kernel(
|
||||
gathered_ptr,
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
cast_to_float32_logits,
|
||||
gathered_softcap_logits,
|
||||
)
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
@@ -1279,6 +1280,9 @@ class LogitsProcessor(nn.Module):
|
||||
logits.shape
|
||||
):
|
||||
assert logits_buffer.dtype == torch.float
|
||||
if logits.is_cuda and logits.dtype in (torch.float16, torch.bfloat16):
|
||||
# Measured on GB300: torch's dtype-casting copy_ is ~2.8x slower.
|
||||
return cast_to_float32_logits(logits, logits_buffer)
|
||||
logits_buffer.copy_(logits)
|
||||
logits = logits_buffer
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user