From 41652225d0e52cbbad63ffd7f052ac19806b027a Mon Sep 17 00:00:00 2001 From: jiaqiang-dot-liu Date: Sun, 11 Oct 2026 16:00:59 +0800 Subject: [PATCH] [Sampler] Sort only the max(top_k) leading columns in the torch sampler (#43346) --- python/sglang/srt/layers/sampler.py | 21 ++- .../srt/sampling/sampling_batch_info.py | 15 +++ .../registered/sampling/test_sampling_mask.py | 1 + .../unit/sampling/test_sampling_batch_info.py | 34 +++++ .../test_torch_sampling_top_k_bound.py | 121 ++++++++++++++++++ 5 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 test/registered/unit/sampling/test_torch_sampling_top_k_bound.py diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 1ab59e5d71..c42140d9fa 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -450,6 +450,7 @@ class Sampler(nn.Module): sampling_info.sampling_seed, positions, return_filtered_probs=return_sampling_mask, + max_top_k=sampling_info.max_top_k, ) if return_sampling_mask: ( @@ -759,6 +760,7 @@ def top_k_top_p_min_p_sampling_from_probs_torch( positions: torch.Tensor, *, return_filtered_probs: bool = False, + max_top_k: Optional[int] = None, ): """ A top-k, top-p and min-p sampling implementation with native pytorch operations. @@ -768,11 +770,26 @@ def top_k_top_p_min_p_sampling_from_probs_torch( By default, returns only sampled token IDs. With return_filtered_probs=True, also returns the actual filtered weights, their token-ID permutation, and the selected weights. + + max_top_k, when given, must be >= every value in top_ks. """ - probs_sort, probs_idx = probs.sort(dim=-1, descending=True) + # The top-k mask below zeroes every column at rank >= top_k, so the max_top_k + # leading columns carry all the mass that survives and sorting only those is + # exact. The seeded path keeps the full sort because the order of tied + # probabilities may depend on k, which is a batch-wide value, and + # deterministic inference must not depend on the batch composition. + if ( + max_top_k is not None + and max_top_k < probs.shape[-1] + and sampling_seed is None + and not return_filtered_probs + ): + probs_sort, probs_idx = torch.topk(probs, k=max_top_k, dim=-1, sorted=True) + else: + probs_sort, probs_idx = probs.sort(dim=-1, descending=True) probs_sum = torch.cumsum(probs_sort, dim=-1) probs_sort[ - torch.arange(0, probs.shape[-1], device=probs.device).view(1, -1) + torch.arange(0, probs_sort.shape[-1], device=probs.device).view(1, -1) >= top_ks.view(-1, 1) ] = 0.0 probs_sort[(probs_sum - probs_sort) > top_ps.view(-1, 1)] = 0.0 diff --git a/python/sglang/srt/sampling/sampling_batch_info.py b/python/sglang/srt/sampling/sampling_batch_info.py index 7d19e77026..b9f5d41acd 100644 --- a/python/sglang/srt/sampling/sampling_batch_info.py +++ b/python/sglang/srt/sampling/sampling_batch_info.py @@ -111,6 +111,11 @@ class SamplingBatchInfo: # device avoids a scalar synchronization in the per-token sampling path. npu_top_k_top_p_eligible: bool = False + # Host-side upper bound on every row's top_k, or None when some row samples + # from the whole vocabulary. Any value >= the true max stays exact, so + # filter_batch keeps it unchanged. + max_top_k: Optional[int] = None + @classmethod def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int): enable_deterministic = get_exec().deterministic.enable_deterministic_inference @@ -263,6 +268,11 @@ class SamplingBatchInfo: npu_top_k_top_p_eligible=all( 1 <= r.sampling_params.top_k <= 1024 for r in reqs ), + max_top_k=( + max(r.sampling_params.top_k for r in reqs) + if reqs and all(1 <= r.sampling_params.top_k < TOP_K_ALL for r in reqs) + else None + ), vocab_size=vocab_size, penalizer_orchestrator=penalizer_orchestrator, has_custom_logit_processor=has_custom_logit_processor, @@ -572,6 +582,11 @@ class SamplingBatchInfo: self.need_top_k_sampling |= other.need_top_k_sampling self.need_min_p_sampling |= other.need_min_p_sampling self.npu_top_k_top_p_eligible &= other.npu_top_k_top_p_eligible + self.max_top_k = ( + None + if self.max_top_k is None or other.max_top_k is None + else max(self.max_top_k, other.max_top_k) + ) self.adjusted_merge_batch(other) diff --git a/test/registered/sampling/test_sampling_mask.py b/test/registered/sampling/test_sampling_mask.py index 7aeec97cc1..d96b0f6683 100644 --- a/test/registered/sampling/test_sampling_mask.py +++ b/test/registered/sampling/test_sampling_mask.py @@ -91,6 +91,7 @@ class TestSamplingMaskCapture(CustomTestCase): top_ks=torch.full((batch_size,), top_k, dtype=torch.int32, device="cuda"), top_ps=torch.full((batch_size,), top_p, device="cuda"), min_ps=torch.full((batch_size,), min_p, device="cuda"), + max_top_k=top_k, sampling_mask_batch_indices=torch.tensor(requested_rows, device="cuda"), ) with patch.object( diff --git a/test/registered/unit/sampling/test_sampling_batch_info.py b/test/registered/unit/sampling/test_sampling_batch_info.py index cfcd9016d4..0ab98310f3 100644 --- a/test/registered/unit/sampling/test_sampling_batch_info.py +++ b/test/registered/unit/sampling/test_sampling_batch_info.py @@ -460,6 +460,16 @@ class TestFilterBatch(CustomTestCase): # logit_bias should also be filtered self.assertEqual(info.logit_bias.shape, (2, VOCAB_SIZE)) + def test_filter_keeps_max_top_k_as_upper_bound(self): + info = _make_info( + batch_size=3, + top_ks=torch.tensor([10, 50, 30], dtype=torch.int32), + max_top_k=50, + ) + info.filter_batch([0, 2], torch.tensor([0, 2])) + self.assertEqual(info.max_top_k, 50) + self.assertGreaterEqual(info.max_top_k, int(info.top_ks.max())) + def test_filter_with_custom_logit_processor(self): proc = MagicMock() info = _make_info(batch_size=3) @@ -604,6 +614,14 @@ class TestMergeBatch(CustomTestCase): self.assertTrue(info1.need_min_p_sampling) # OR semantics self.assertFalse(info1.npu_top_k_top_p_eligible) # AND semantics + def test_merge_max_top_k(self): + cases = (((20, 50), 50), ((50, 20), 50), ((20, None), None), ((None, 20), None)) + for (lhs, rhs), expected in cases: + with self.subTest(lhs=lhs, rhs=rhs): + info1 = _make_info(max_top_k=lhs) + info1.merge_batch(_make_info(max_top_k=rhs)) + self.assertEqual(info1.max_top_k, expected) + def test_merge_with_logit_bias(self): """Test that merge pads missing logit_bias with zeros before concatenation.""" info1 = _make_info(batch_size=1) @@ -792,6 +810,22 @@ class TestFromScheduleBatch(CustomTestCase): self.assertEqual(info.npu_top_k_top_p_eligible, expected) + def test_max_top_k_uses_request_params(self): + cases = ( + ((20, 50), 50), + ((1, 20), 20), + ((20, TOP_K_ALL), None), + ) + for top_ks, expected in cases: + with self.subTest(top_ks=top_ks): + batch = MagicMock() + batch.reqs = [self._make_req(top_k=top_k) for top_k in top_ks] + batch.device = DEVICE + + info = SamplingBatchInfo.from_schedule_batch(batch, VOCAB_SIZE) + + self.assertEqual(info.max_top_k, expected) + def test_no_logit_bias_when_all_none(self): """Test that logit_bias stays None when no request has logit_bias set.""" diff --git a/test/registered/unit/sampling/test_torch_sampling_top_k_bound.py b/test/registered/unit/sampling/test_torch_sampling_top_k_bound.py new file mode 100644 index 0000000000..63317d2214 --- /dev/null +++ b/test/registered/unit/sampling/test_torch_sampling_top_k_bound.py @@ -0,0 +1,121 @@ +"""The bounded top-k sort in the torch sampler must match the full vocabulary sort.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +import unittest +from unittest import mock + +import torch + +from sglang.srt.layers.sampler import top_k_top_p_min_p_sampling_from_probs_torch +from sglang.test.test_utils import CustomTestCase + +VOCAB_SIZE = 4096 +BATCH_SIZE = 8 + + +def _probs(seed: int) -> torch.Tensor: + gen = torch.Generator().manual_seed(seed) + return torch.softmax( + torch.randn(BATCH_SIZE, VOCAB_SIZE, generator=gen) * 3.0, dim=-1 + ) + + +def _run(probs, top_ks, top_ps, min_ps, need_min_p, max_top_k): + """Return (token ids, the weights handed to multinomial). + + multinomial is replaced by argmax so both sorts can be compared on the same draw. + """ + seen = {} + + def _argmax(weights, num_samples): + seen["weights"] = weights.clone() + return weights.argmax(dim=-1, keepdim=True) + + with mock.patch("torch.multinomial", side_effect=_argmax): + token_ids = top_k_top_p_min_p_sampling_from_probs_torch( + probs.clone(), + top_ks, + top_ps, + min_ps, + need_min_p, + None, + torch.zeros(BATCH_SIZE, dtype=torch.int64), + max_top_k=max_top_k, + ) + return token_ids, seen["weights"] + + +class TestTorchSamplingTopKBound(CustomTestCase): + def test_bounded_sort_matches_full_sort(self): + cases = ( + # (top_ks, top_ps, min_ps, need_min_p) + ([20] * BATCH_SIZE, [1.0] * BATCH_SIZE, [0.0] * BATCH_SIZE, False), + ( + [1, 2, 5, 20, 50, 100, 7, 64], + [1.0] * BATCH_SIZE, + [0.0] * BATCH_SIZE, + False, + ), + ([20] * BATCH_SIZE, [0.95, 0.5, 0.8, 1.0] * 2, [0.0] * BATCH_SIZE, False), + ([40] * BATCH_SIZE, [0.9] * BATCH_SIZE, [0.0, 0.05] * 4, True), + ) + for i, (top_ks, top_ps, min_ps, need_min_p) in enumerate(cases): + with self.subTest(case=i): + probs = _probs(seed=i) + top_ks = torch.tensor(top_ks, dtype=torch.int32) + top_ps = torch.tensor(top_ps, dtype=torch.float32) + min_ps = torch.tensor(min_ps, dtype=torch.float32) + max_top_k = int(top_ks.max()) + + full_ids, full_weights = _run( + probs, top_ks, top_ps, min_ps, need_min_p, max_top_k=None + ) + bounded_ids, bounded_weights = _run( + probs, top_ks, top_ps, min_ps, need_min_p, max_top_k=max_top_k + ) + + self.assertEqual(full_weights.shape[-1], VOCAB_SIZE) + self.assertEqual(bounded_weights.shape[-1], max_top_k) + self.assertTrue(torch.all(full_weights[:, max_top_k:] == 0)) + torch.testing.assert_close( + bounded_weights, full_weights[:, :max_top_k], rtol=0, atol=0 + ) + torch.testing.assert_close(bounded_ids, full_ids, rtol=0, atol=0) + + def test_return_filtered_probs_keeps_full_sort(self): + probs = _probs(seed=0) + top_ks = torch.full((BATCH_SIZE,), 20, dtype=torch.int32) + + _, filtered_probs, token_ids, _ = top_k_top_p_min_p_sampling_from_probs_torch( + probs.clone(), + top_ks, + torch.ones(BATCH_SIZE), + torch.zeros(BATCH_SIZE), + False, + None, + torch.zeros(BATCH_SIZE, dtype=torch.int64), + return_filtered_probs=True, + max_top_k=20, + ) + self.assertEqual(filtered_probs.shape[-1], VOCAB_SIZE) + self.assertEqual(token_ids.shape[-1], VOCAB_SIZE) + + def test_bound_at_or_above_vocab_keeps_full_sort(self): + probs = _probs(seed=0) + top_ks = torch.full((BATCH_SIZE,), VOCAB_SIZE, dtype=torch.int32) + _, weights = _run( + probs, + top_ks, + torch.ones(BATCH_SIZE), + torch.zeros(BATCH_SIZE), + False, + max_top_k=VOCAB_SIZE, + ) + self.assertEqual(weights.shape[-1], VOCAB_SIZE) + + +if __name__ == "__main__": + unittest.main()