[Perf] logits: Write lm_head logits straight into the fp32 logits buffer (#45631)

This commit is contained in:
Khoa Pham
2026-10-21 20:32:16 -07:00
committed by GitHub
parent 1962cbeebe
commit f1e26dc658
3 changed files with 155 additions and 28 deletions
+33 -20
View File
@@ -912,8 +912,15 @@ class LogitsProcessor(nn.Module):
if self.do_tensor_parallel_all_gather_dp_attn
else LoRABatchLayout.DP_LOCAL
)
logits_buffer = (
logits_metadata.next_token_logits_buffer
if use_logits_buffer and not self.do_tensor_parallel_all_gather
else None
)
with get_forward().scoped(lora_batch_layout=lora_batch_layout):
logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias)
logits = self._compute_lm_head(
hidden_states, lm_head, embedding_bias, out=logits_buffer
)
_trace_e2e_logits("lm_head_returned", logits_shape=tuple(logits.shape))
if envs.SGLANG_TRACE_LOGITS_E2E_SYNC.get():
_trace_e2e_logits("post_lm_head_sync_enter")
@@ -1061,6 +1068,8 @@ class LogitsProcessor(nn.Module):
hidden_states: torch.Tensor,
lm_head: VocabParallelEmbedding,
embedding_bias: Optional[torch.Tensor] = None,
*,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
quant_method = getattr(lm_head, "quant_method", None)
if hasattr(lm_head, "set_lora") and hasattr(lm_head, "apply_lora"):
@@ -1070,26 +1079,28 @@ class LogitsProcessor(nn.Module):
logits = quant_method.apply(lm_head, hidden_states, embedding_bias)
elif hasattr(lm_head, "weight"):
# Normal linear layer
if self.use_fp32_lm_head:
# Avoid materializing FP32 copies for same-dtype CUDA FP16/BF16
# inputs. Retain explicit FP32 casts for unsupported devices or
# dtype combinations.
use_mm_out_dtype = (
hidden_states.is_cuda
and hidden_states.dtype == lm_head.weight.dtype
and hidden_states.dtype in (torch.float16, torch.bfloat16)
if out is not None and out.shape != (
hidden_states.shape[0],
lm_head.weight.shape[0],
):
out = None
use_mm_out_dtype = (
hidden_states.is_cuda
and hidden_states.dtype == lm_head.weight.dtype
and hidden_states.dtype in (torch.float16, torch.bfloat16)
)
if use_mm_out_dtype and (self.use_fp32_lm_head or out is not None):
logits = torch.mm(
hidden_states,
lm_head.weight.T,
out_dtype=torch.float32,
out=out,
)
elif self.use_fp32_lm_head:
logits = torch.matmul(
hidden_states.to(torch.float32),
lm_head.weight.to(torch.float32).T,
)
if use_mm_out_dtype:
logits = torch.mm(
hidden_states,
lm_head.weight.T,
out_dtype=torch.float32,
)
else:
logits = torch.matmul(
hidden_states.to(torch.float32),
lm_head.weight.to(torch.float32).T,
)
elif use_intel_amx_backend(lm_head):
logits = torch.ops.sgl_kernel.weight_packed_linear(
hidden_states.to(lm_head.weight.dtype),
@@ -1248,6 +1259,8 @@ class LogitsProcessor(nn.Module):
softcap: Optional[float] = None,
) -> torch.Tensor:
logits_buffer = logits_metadata.next_token_logits_buffer if use_buffer else None
if logits is logits_buffer and softcap is None:
return logits
if logits.shape[-1] > self.vocab_size:
logits = logits[:, : self.vocab_size]
logits_width = logits.shape[-1]
+12 -2
View File
@@ -111,9 +111,13 @@ class MuseSpark1xLogitsProcessor(OSSLogitsProcessor):
super().__init__(config, **kwargs)
self.use_fp32_output = True
def _compute_lm_head(self, hidden_states, lm_head, embedding_bias=None):
def _compute_lm_head(
self, hidden_states, lm_head, embedding_bias=None, *, out=None
):
if not hasattr(lm_head, "weight"):
return super()._compute_lm_head(hidden_states, lm_head, embedding_bias)
return super()._compute_lm_head(
hidden_states, lm_head, embedding_bias, out=out
)
output_dtype = torch.float32 if self.use_fp32_output else lm_head.weight.dtype
if use_intel_amx_backend(lm_head):
# ParallelLMHead swaps in PackWeightMethod on CPU+AMX, which repacks
@@ -126,10 +130,16 @@ class MuseSpark1xLogitsProcessor(OSSLogitsProcessor):
True,
).to(output_dtype)
else:
if out is not None and out.shape != (
hidden_states.shape[0],
lm_head.weight.shape[0],
):
out = None
logits = torch.mm(
hidden_states.to(lm_head.weight.dtype),
lm_head.weight.T,
out_dtype=output_dtype,
out=out,
)
if embedding_bias is not None:
logits = logits + embedding_bias
+110 -6
View File
@@ -39,7 +39,13 @@ class TestLMHeadFP32(CustomTestCase):
):
raise unittest.SkipTest("needs CUDA GPU or XPU")
def _make_logprocessor(self, vocab_size, enable_fp32, config_enable_fp32=False):
def _make_logprocessor(
self,
vocab_size,
enable_fp32,
config_enable_fp32=False,
processor_cls=LogitsProcessor,
):
# LogitsProcessor reads get_exec().features.enable_fp32_lm_head
# from the published config.
override = get_context().override_server_args(enable_fp32_lm_head=enable_fp32)
@@ -50,8 +56,9 @@ class TestLMHeadFP32(CustomTestCase):
final_logit_softcapping=None,
enable_lm_head_fp32=config_enable_fp32,
)
return LogitsProcessor(cfg, skip_all_gather=True, logit_scale=None)
return processor_cls(cfg, skip_all_gather=True, logit_scale=None)
@torch.inference_mode()
def _run_case(
self,
hidden_state_dtype,
@@ -123,10 +130,15 @@ class TestLMHeadFP32(CustomTestCase):
self.assertEqual(state["operation"], expected_operation)
self.assertEqual(state["a"], expected_a_dtype)
self.assertEqual(state["b"], expected_b_dtype)
self.assertEqual(
state["out_dtype"],
torch.float32 if expected_operation == "mm" else None,
expected_out_dtype = torch.float32 if expected_operation == "mm" else None
self.assertEqual(state["out_dtype"], expected_out_dtype)
reference = F.linear(
hidden_state.to(expected_a_dtype).double(),
head.weight.to(expected_b_dtype).double(),
)
if expected_out_dtype is None:
reference = reference.to(expected_a_dtype)
torch.testing.assert_close(logits, reference.float(), rtol=1e-5, atol=1e-5)
def test_flag_true_fp16_activations(self):
expected_operation = "mm" if torch.cuda.is_available() else "matmul"
@@ -168,7 +180,12 @@ class TestLMHeadFP32(CustomTestCase):
def test_flag_false_fp16_path(self):
self._run_case(
torch.float16, False, torch.float16, torch.float16, torch.float16, "matmul"
torch.float16,
False,
torch.float16,
torch.float16,
torch.float16,
"matmul",
)
def test_flag_false_bf16_path(self):
@@ -181,6 +198,93 @@ class TestLMHeadFP32(CustomTestCase):
"matmul",
)
@torch.inference_mode()
def test_logits_buffer_shapes_and_transforms(self):
if not torch.cuda.is_available():
self.skipTest("requires CUDA fp32 GEMM output")
from sglang.srt.models.muse_spark import MuseSpark1xLogitsProcessor
cases = [
(128, 2, True, None, None),
(129, 2, True, None, None),
(128, 3, True, None, None),
(128, 2, False, None, None),
(128, 2, True, 0.5, None),
(128, 2, True, None, 3.0),
(129, 2, True, 0.5, 3.0),
]
for processor_cls in (LogitsProcessor, MuseSpark1xLogitsProcessor):
for dtype in (torch.bfloat16, torch.float16):
for vocab, rows, use_buffer, scale, softcap in cases:
with self.subTest(
processor=processor_cls.__name__,
dtype=dtype,
case=(vocab, rows, use_buffer, scale, softcap),
):
processor = self._make_logprocessor(
vocab, False, processor_cls=processor_cls
)
processor.logit_scale = scale
processor.final_logit_softcapping = softcap
padded_vocab = ((vocab + 127) // 128) * 128
head = LMHeadStub(padded_vocab, 64, dtype)
hidden = torch.randn(2, 64, device=get_device(), dtype=dtype)
buffer = torch.full(
(rows, vocab), float("nan"), device=get_device()
)
meta = DummyMeta()
meta.next_token_logits_buffer = buffer
logits = processor._get_logits(
hidden, head, meta, use_logits_buffer=use_buffer
)
reference = F.linear(hidden.double(), head.weight.double())[
:, :vocab
]
direct = use_buffer and rows == 2 and padded_vocab == vocab
reference = reference.to(
torch.float32
if direct or processor_cls is MuseSpark1xLogitsProcessor
else dtype
)
if scale is not None:
reference *= scale
reference = reference.float()
if softcap is not None:
reference = softcap * torch.tanh(reference / softcap)
torch.testing.assert_close(
logits, reference, rtol=1e-5, atol=1e-5
)
self.assertEqual(buffer.shape, (rows, vocab))
if use_buffer and rows == 2:
self.assertIs(logits, buffer)
else:
self.assertTrue(torch.isnan(buffer).all())
@torch.inference_mode()
def test_logits_buffer_cuda_graph_replay(self):
if not torch.cuda.is_available():
self.skipTest("requires CUDA graphs")
processor = self._make_logprocessor(128, False)
hidden = torch.randn(2, 64, dtype=torch.bfloat16, device="cuda")
head = LMHeadStub(128, 64, torch.bfloat16)
meta = DummyMeta()
meta.next_token_logits_buffer = torch.empty(2, 128, device="cuda")
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
processor._get_logits(hidden, head, meta)
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
logits = processor._get_logits(hidden, head, meta)
for _ in range(3):
hidden.normal_()
graph.replay()
reference = F.linear(hidden.double(), head.weight.double()).float()
torch.testing.assert_close(logits, reference, rtol=1e-5, atol=1e-5)
self.assertIs(logits, meta.next_token_logits_buffer)
def test_model_config_enables_fp32_without_server_flag(self):
expected_operation = "mm" if torch.cuda.is_available() else "matmul"
expected_dtype = (