diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 26c64dfee0..dfe56a17f2 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -912,8 +912,15 @@ class LogitsProcessor(nn.Module): if self.do_tensor_parallel_all_gather_dp_attn else LoRABatchLayout.DP_LOCAL ) + logits_buffer = ( + logits_metadata.next_token_logits_buffer + if use_logits_buffer and not self.do_tensor_parallel_all_gather + else None + ) with get_forward().scoped(lora_batch_layout=lora_batch_layout): - logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias) + logits = self._compute_lm_head( + hidden_states, lm_head, embedding_bias, out=logits_buffer + ) _trace_e2e_logits("lm_head_returned", logits_shape=tuple(logits.shape)) if envs.SGLANG_TRACE_LOGITS_E2E_SYNC.get(): _trace_e2e_logits("post_lm_head_sync_enter") @@ -1061,6 +1068,8 @@ class LogitsProcessor(nn.Module): hidden_states: torch.Tensor, lm_head: VocabParallelEmbedding, embedding_bias: Optional[torch.Tensor] = None, + *, + out: Optional[torch.Tensor] = None, ) -> torch.Tensor: quant_method = getattr(lm_head, "quant_method", None) if hasattr(lm_head, "set_lora") and hasattr(lm_head, "apply_lora"): @@ -1070,26 +1079,28 @@ class LogitsProcessor(nn.Module): logits = quant_method.apply(lm_head, hidden_states, embedding_bias) elif hasattr(lm_head, "weight"): # Normal linear layer - if self.use_fp32_lm_head: - # Avoid materializing FP32 copies for same-dtype CUDA FP16/BF16 - # inputs. Retain explicit FP32 casts for unsupported devices or - # dtype combinations. - use_mm_out_dtype = ( - hidden_states.is_cuda - and hidden_states.dtype == lm_head.weight.dtype - and hidden_states.dtype in (torch.float16, torch.bfloat16) + if out is not None and out.shape != ( + hidden_states.shape[0], + lm_head.weight.shape[0], + ): + out = None + use_mm_out_dtype = ( + hidden_states.is_cuda + and hidden_states.dtype == lm_head.weight.dtype + and hidden_states.dtype in (torch.float16, torch.bfloat16) + ) + if use_mm_out_dtype and (self.use_fp32_lm_head or out is not None): + logits = torch.mm( + hidden_states, + lm_head.weight.T, + out_dtype=torch.float32, + out=out, + ) + elif self.use_fp32_lm_head: + logits = torch.matmul( + hidden_states.to(torch.float32), + lm_head.weight.to(torch.float32).T, ) - if use_mm_out_dtype: - logits = torch.mm( - hidden_states, - lm_head.weight.T, - out_dtype=torch.float32, - ) - else: - logits = torch.matmul( - hidden_states.to(torch.float32), - lm_head.weight.to(torch.float32).T, - ) elif use_intel_amx_backend(lm_head): logits = torch.ops.sgl_kernel.weight_packed_linear( hidden_states.to(lm_head.weight.dtype), @@ -1248,6 +1259,8 @@ class LogitsProcessor(nn.Module): softcap: Optional[float] = None, ) -> torch.Tensor: logits_buffer = logits_metadata.next_token_logits_buffer if use_buffer else None + if logits is logits_buffer and softcap is None: + return logits if logits.shape[-1] > self.vocab_size: logits = logits[:, : self.vocab_size] logits_width = logits.shape[-1] diff --git a/python/sglang/srt/models/muse_spark.py b/python/sglang/srt/models/muse_spark.py index 35a458bedf..063861b339 100644 --- a/python/sglang/srt/models/muse_spark.py +++ b/python/sglang/srt/models/muse_spark.py @@ -111,9 +111,13 @@ class MuseSpark1xLogitsProcessor(OSSLogitsProcessor): super().__init__(config, **kwargs) self.use_fp32_output = True - def _compute_lm_head(self, hidden_states, lm_head, embedding_bias=None): + def _compute_lm_head( + self, hidden_states, lm_head, embedding_bias=None, *, out=None + ): if not hasattr(lm_head, "weight"): - return super()._compute_lm_head(hidden_states, lm_head, embedding_bias) + return super()._compute_lm_head( + hidden_states, lm_head, embedding_bias, out=out + ) output_dtype = torch.float32 if self.use_fp32_output else lm_head.weight.dtype if use_intel_amx_backend(lm_head): # ParallelLMHead swaps in PackWeightMethod on CPU+AMX, which repacks @@ -126,10 +130,16 @@ class MuseSpark1xLogitsProcessor(OSSLogitsProcessor): True, ).to(output_dtype) else: + if out is not None and out.shape != ( + hidden_states.shape[0], + lm_head.weight.shape[0], + ): + out = None logits = torch.mm( hidden_states.to(lm_head.weight.dtype), lm_head.weight.T, out_dtype=output_dtype, + out=out, ) if embedding_bias is not None: logits = logits + embedding_bias diff --git a/test/registered/rl/test_fp32_lm_head.py b/test/registered/rl/test_fp32_lm_head.py index 2e0f1676b0..6cdde78d7b 100644 --- a/test/registered/rl/test_fp32_lm_head.py +++ b/test/registered/rl/test_fp32_lm_head.py @@ -39,7 +39,13 @@ class TestLMHeadFP32(CustomTestCase): ): raise unittest.SkipTest("needs CUDA GPU or XPU") - def _make_logprocessor(self, vocab_size, enable_fp32, config_enable_fp32=False): + def _make_logprocessor( + self, + vocab_size, + enable_fp32, + config_enable_fp32=False, + processor_cls=LogitsProcessor, + ): # LogitsProcessor reads get_exec().features.enable_fp32_lm_head # from the published config. override = get_context().override_server_args(enable_fp32_lm_head=enable_fp32) @@ -50,8 +56,9 @@ class TestLMHeadFP32(CustomTestCase): final_logit_softcapping=None, enable_lm_head_fp32=config_enable_fp32, ) - return LogitsProcessor(cfg, skip_all_gather=True, logit_scale=None) + return processor_cls(cfg, skip_all_gather=True, logit_scale=None) + @torch.inference_mode() def _run_case( self, hidden_state_dtype, @@ -123,10 +130,15 @@ class TestLMHeadFP32(CustomTestCase): self.assertEqual(state["operation"], expected_operation) self.assertEqual(state["a"], expected_a_dtype) self.assertEqual(state["b"], expected_b_dtype) - self.assertEqual( - state["out_dtype"], - torch.float32 if expected_operation == "mm" else None, + expected_out_dtype = torch.float32 if expected_operation == "mm" else None + self.assertEqual(state["out_dtype"], expected_out_dtype) + reference = F.linear( + hidden_state.to(expected_a_dtype).double(), + head.weight.to(expected_b_dtype).double(), ) + if expected_out_dtype is None: + reference = reference.to(expected_a_dtype) + torch.testing.assert_close(logits, reference.float(), rtol=1e-5, atol=1e-5) def test_flag_true_fp16_activations(self): expected_operation = "mm" if torch.cuda.is_available() else "matmul" @@ -168,7 +180,12 @@ class TestLMHeadFP32(CustomTestCase): def test_flag_false_fp16_path(self): self._run_case( - torch.float16, False, torch.float16, torch.float16, torch.float16, "matmul" + torch.float16, + False, + torch.float16, + torch.float16, + torch.float16, + "matmul", ) def test_flag_false_bf16_path(self): @@ -181,6 +198,93 @@ class TestLMHeadFP32(CustomTestCase): "matmul", ) + @torch.inference_mode() + def test_logits_buffer_shapes_and_transforms(self): + if not torch.cuda.is_available(): + self.skipTest("requires CUDA fp32 GEMM output") + from sglang.srt.models.muse_spark import MuseSpark1xLogitsProcessor + + cases = [ + (128, 2, True, None, None), + (129, 2, True, None, None), + (128, 3, True, None, None), + (128, 2, False, None, None), + (128, 2, True, 0.5, None), + (128, 2, True, None, 3.0), + (129, 2, True, 0.5, 3.0), + ] + for processor_cls in (LogitsProcessor, MuseSpark1xLogitsProcessor): + for dtype in (torch.bfloat16, torch.float16): + for vocab, rows, use_buffer, scale, softcap in cases: + with self.subTest( + processor=processor_cls.__name__, + dtype=dtype, + case=(vocab, rows, use_buffer, scale, softcap), + ): + processor = self._make_logprocessor( + vocab, False, processor_cls=processor_cls + ) + processor.logit_scale = scale + processor.final_logit_softcapping = softcap + padded_vocab = ((vocab + 127) // 128) * 128 + head = LMHeadStub(padded_vocab, 64, dtype) + hidden = torch.randn(2, 64, device=get_device(), dtype=dtype) + buffer = torch.full( + (rows, vocab), float("nan"), device=get_device() + ) + meta = DummyMeta() + meta.next_token_logits_buffer = buffer + logits = processor._get_logits( + hidden, head, meta, use_logits_buffer=use_buffer + ) + reference = F.linear(hidden.double(), head.weight.double())[ + :, :vocab + ] + direct = use_buffer and rows == 2 and padded_vocab == vocab + reference = reference.to( + torch.float32 + if direct or processor_cls is MuseSpark1xLogitsProcessor + else dtype + ) + if scale is not None: + reference *= scale + reference = reference.float() + if softcap is not None: + reference = softcap * torch.tanh(reference / softcap) + torch.testing.assert_close( + logits, reference, rtol=1e-5, atol=1e-5 + ) + self.assertEqual(buffer.shape, (rows, vocab)) + if use_buffer and rows == 2: + self.assertIs(logits, buffer) + else: + self.assertTrue(torch.isnan(buffer).all()) + + @torch.inference_mode() + def test_logits_buffer_cuda_graph_replay(self): + if not torch.cuda.is_available(): + self.skipTest("requires CUDA graphs") + processor = self._make_logprocessor(128, False) + hidden = torch.randn(2, 64, dtype=torch.bfloat16, device="cuda") + head = LMHeadStub(128, 64, torch.bfloat16) + meta = DummyMeta() + meta.next_token_logits_buffer = torch.empty(2, 128, device="cuda") + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + processor._get_logits(hidden, head, meta) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + logits = processor._get_logits(hidden, head, meta) + for _ in range(3): + hidden.normal_() + graph.replay() + reference = F.linear(hidden.double(), head.weight.double()).float() + torch.testing.assert_close(logits, reference, rtol=1e-5, atol=1e-5) + self.assertIs(logits, meta.next_token_logits_buffer) + def test_model_config_enables_fp32_without_server_flag(self): expected_operation = "mm" if torch.cuda.is_available() else "matmul" expected_dtype = (