From d669efba8464538acf8b2ee9417d4a897d00221a Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Thu, 1 Oct 2026 18:35:28 +0800 Subject: [PATCH] [AMD][V4.1][*/N] Greedy dspark draft/accept under SGLANG_SIMULATE_ACC_LEN (#41981) Co-authored-by: wunhuang Co-authored-by: Cursor --- .../ops/speculative/dspark/simulated_bonus.py | 104 ++++++++++++++++++ python/sglang/srt/environ.py | 3 + .../dspark_components/dspark_verify.py | 47 +++++++- .../dspark_components/dspark_worker_v2.py | 23 ++++ .../dspark/test_dspark_simulated_bonus.py | 79 +++++++++++++ 5 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py create mode 100644 test/registered/spec/dspark/test_dspark_simulated_bonus.py diff --git a/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py b/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py new file mode 100644 index 0000000000..39b72d86a8 --- /dev/null +++ b/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import random + +import torch +import triton +import triton.language as tl + + +@triton.jit(do_not_specialize=["seed"]) +def _simulated_bonus_partial_kernel( + logits_ptr, + logits_stride, + temperatures_ptr, + part_val_ptr, + part_idx_ptr, + seed, + vocab, + ROWS: tl.constexpr, + NSPLIT: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + split = tl.program_id(1) + t = tl.load(temperatures_ptr + row // ROWS).to(tl.float32) + inv_t = 1.0 / tl.maximum(t, 1e-5) + offs = split * BLOCK + tl.arange(0, BLOCK) + mask = offs < vocab + x = tl.load( + logits_ptr + row.to(tl.int64) * logits_stride + offs, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + u = tl.rand(seed, row * vocab + offs) + v = tl.where(mask, x * inv_t - tl.log(-tl.log(u)), float("-inf")) + top = tl.max(v, axis=0) + idx = tl.min(tl.where(v == top, offs, vocab), axis=0) + tl.store(part_val_ptr + row * NSPLIT + split, top) + tl.store(part_idx_ptr + row * NSPLIT + split, idx) + + +@triton.jit +def _simulated_bonus_select_kernel( + part_val_ptr, + part_idx_ptr, + correct_len_ptr, + out_ptr, + vocab, + ROWS: tl.constexpr, + NSPLIT: tl.constexpr, +): + req = tl.program_id(0) + row = req * ROWS + tl.load(correct_len_ptr + req) + s = tl.arange(0, NSPLIT) + val = tl.load(part_val_ptr + row * NSPLIT + s) + idx = tl.load(part_idx_ptr + row * NSPLIT + s) + top = tl.max(val, axis=0) + tl.store( + out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64) + ) + + +def simulated_bonus_sample( + *, + target_logits: torch.Tensor, + correct_len: torch.Tensor, + temperatures: torch.Tensor, + bs: int, + rows_per_request: int, +) -> torch.Tensor: + """Gumbel-max temperature sample of every verify row; returns, per request, the + sample of the row at ``correct_len``. [bs] int64.""" + assert target_logits.stride(1) == 1 + device = target_logits.device + vocab = target_logits.shape[1] + block = 4096 + nsplit = triton.next_power_of_2(triton.cdiv(vocab, block)) + rows = bs * rows_per_request + part_val = torch.empty((rows, nsplit), dtype=torch.float32, device=device) + part_idx = torch.empty((rows, nsplit), dtype=torch.int32, device=device) + _simulated_bonus_partial_kernel[(rows, nsplit)]( + target_logits, + target_logits.stride(0), + temperatures.view(-1), + part_val, + part_idx, + random.getrandbits(31), + vocab, + ROWS=rows_per_request, + NSPLIT=nsplit, + BLOCK=block, + num_warps=8, + ) + out = torch.empty((bs,), dtype=torch.int64, device=device) + _simulated_bonus_select_kernel[(bs,)]( + part_val, + part_idx, + correct_len.contiguous(), + out, + vocab, + ROWS=rows_per_request, + NSPLIT=nsplit, + ) + return out diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d4dc410de8..5a9a98b184 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -521,6 +521,9 @@ class Envs: SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1) SGLANG_SIMULATE_ACC_METHOD = EnvStr("match-expected") SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr("fixed") + # DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests + # run the greedy draft/accept path and only the bonus token is temperature-sampled. + SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True) SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False) SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index dd45c0d90c..4a1615c5f0 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -26,6 +26,9 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( build_unified_commit_inject_layout, scatter_compact_to_strided_into, ) +from sglang.kernels.ops.speculative.dspark.simulated_bonus import ( + simulated_bonus_sample, +) from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode @@ -46,9 +49,10 @@ from sglang.srt.speculative.spec_utils import ( SIMULATE_ACC_METHOD, sample_simulated_acc_len, ) -from sglang.srt.utils import is_npu +from sglang.srt.utils import is_hip, is_npu from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect +_is_hip = is_hip() _is_npu = is_npu() # Draft proposal probs feeding rejection sampling; the data layer is the @@ -148,6 +152,7 @@ class TargetVerifyExecutor: layout: Optional[RaggedVerifyLayout], prefix_lens: torch.Tensor, draft_tokens: torch.Tensor, + simulate_bonus_sampling_info=None, ) -> AcceptOuts: """Produce the per-request accept outcome after target verify. @@ -159,25 +164,38 @@ class TargetVerifyExecutor: if folded_accept: return self.verify_epilogue.read_accept(bs) + simulate = self._simulate_acc_len > 0 + # Simulated acceptance overwrites correct_len below, so a sampling accept + # (draft/target softmax + rejection) would be computed only to be discarded. + accept_sampling_info = None if simulate and _is_hip else sampling_info correct_len, bonus, cap_trim_lens = accept_draft_tokens( candidates=verify_ids_2d, target_logits=target_logits, draft_block=draft_block, - sampling_info=sampling_info, + sampling_info=accept_sampling_info, draft_input=draft_input, gamma=self.gamma, verify_num_draft_tokens=self.verify_num_draft_tokens, cutoff_layout=layout, fused_argmax=self._target_is_dsv41, ) - if self._simulate_acc_len > 0: + if simulate: correct_len = self._simulated_correct_len( bs=bs, dtype=correct_len.dtype, device=correct_len.device ) + if simulate_bonus_sampling_info is not None: + bonus = sample_simulated_bonus( + target_logits=target_logits, + correct_len=correct_len, + greedy_bonus=bonus, + sampling_info=simulate_bonus_sampling_info, + bs=bs, + verify_num_draft_tokens=self.verify_num_draft_tokens, + ) site = ( SpecTpSyncSite.DSPARK_ACCEPT_GREEDY - if sampling_info is None or sampling_info.is_all_greedy + if accept_sampling_info is None or accept_sampling_info.is_all_greedy else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE ) self._tp_sync.sync(site, correct_len) @@ -811,6 +829,27 @@ class DsparkVerifyEpilogue: ) +def sample_simulated_bonus( + *, + target_logits: torch.Tensor, + correct_len: torch.Tensor, + greedy_bonus: torch.Tensor, + sampling_info, + bs: int, + verify_num_draft_tokens: int, +) -> torch.Tensor: + """Temperature-sample every verify row and return the row at the (simulated) + correct_len as the bonus token. Temperature only: callers exclude top-p/top-k/min-p.""" + bonus = simulated_bonus_sample( + target_logits=target_logits, + correct_len=correct_len, + temperatures=sampling_info.temperatures, + bs=bs, + rows_per_request=verify_num_draft_tokens, + ) + return bonus.to(greedy_bonus.dtype).view_as(greedy_bonus) + + def accept_draft_tokens( *, candidates: torch.Tensor, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 6dad00afa3..99d8327d47 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -85,6 +85,7 @@ from sglang.srt.speculative.spec_utils import ( from sglang.srt.utils import ( is_cuda, is_cuda_alike, + is_hip, is_npu, is_pin_memory_available, ) @@ -94,6 +95,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_is_hip = is_hip() _is_npu = is_npu() @@ -365,6 +367,11 @@ class DSparkWorkerV2(BaseSpecWorker): ) self._simulate_acc_len = float(envs.SGLANG_SIMULATE_ACC_LEN.get()) + self._simulate_acc_greedy = ( + _is_hip + and self._simulate_acc_len > 0 + and envs.SGLANG_SIMULATE_ACC_GREEDY.get() + ) if ( self._simulate_acc_len > 0 and self._simulate_acc_len != 1.0 @@ -839,6 +846,21 @@ class DSparkWorkerV2(BaseSpecWorker): ) sampling_info = batch.sampling_info + # Simulated acceptance overrides correct_len, so only the bonus token needs the + # request's temperature: draft + accept run greedy, the bonus is still sampled. + simulate_bonus_sampling_info = None + if ( + self._simulate_acc_greedy + and sampling_info is not None + and not sampling_info.is_all_greedy + and not batch.has_grammar + and verify_logits_adjustments_are_noop(sampling_info) + and not sampling_info.need_top_p_sampling + and not sampling_info.need_top_k_sampling + and not sampling_info.need_min_p_sampling + ): + simulate_bonus_sampling_info = sampling_info + sampling_info = None with self._draft_context(), self._observers.segment(InfoSegment.DRAFT): proposal = self._proposer.propose( batch=batch, @@ -963,6 +985,7 @@ class DSparkWorkerV2(BaseSpecWorker): layout=layout, prefix_lens=prefix_lens, draft_tokens=draft_tokens, + simulate_bonus_sampling_info=simulate_bonus_sampling_info, ) self.model_runner.ngram_embedding_manager.update_after_verify( verify_ids_2d=verify_ids_2d, diff --git a/test/registered/spec/dspark/test_dspark_simulated_bonus.py b/test/registered/spec/dspark/test_dspark_simulated_bonus.py new file mode 100644 index 0000000000..202babedae --- /dev/null +++ b/test/registered/spec/dspark/test_dspark_simulated_bonus.py @@ -0,0 +1,79 @@ +import types +import unittest + +import torch + +from sglang.srt.speculative.dspark_components.dspark_verify import ( + sample_simulated_bonus, +) +from sglang.test.ci.ci_register import register_amd_ci + +register_amd_ci(est_time=10, suite="stage-b-test-1-gpu-small-amd") + + +def _info(temps): + return types.SimpleNamespace(temperatures=torch.tensor(temps, device="cuda")) + + +class TestSimulatedBonus(unittest.TestCase): + def test_picks_row_at_correct_len(self): + torch.manual_seed(0) + bs, rows, vocab = 3, 6, 1000 + logits = torch.randn(bs * rows, vocab, device="cuda") + correct_len = torch.tensor([0, 5, 2], device="cuda", dtype=torch.int32) + greedy_bonus = torch.zeros(bs, dtype=torch.int64, device="cuda") + out = sample_simulated_bonus( + target_logits=logits, + correct_len=correct_len, + greedy_bonus=greedy_bonus, + sampling_info=_info([1e-6] * bs), + bs=bs, + verify_num_draft_tokens=rows, + ) + ref = logits.view(bs, rows, vocab).argmax(-1)[ + torch.arange(bs), correct_len.long() + ] + self.assertEqual(out.dtype, torch.int64) + self.assertTrue(torch.equal(out, ref)) + + def test_bf16_full_vocab_low_temperature(self): + torch.manual_seed(2) + bs, rows, vocab = 8, 6, 129280 + logits = torch.randn(bs * rows, vocab, device="cuda", dtype=torch.bfloat16) + correct_len = torch.randint(0, rows, (bs,), device="cuda", dtype=torch.int64) + out = sample_simulated_bonus( + target_logits=logits, + correct_len=correct_len, + greedy_bonus=torch.zeros(bs, dtype=torch.int64, device="cuda"), + sampling_info=_info([1e-6] * bs), + bs=bs, + verify_num_draft_tokens=rows, + ) + ref = ( + logits.float() + .view(bs, rows, vocab) + .argmax(-1)[torch.arange(bs), correct_len] + ) + self.assertTrue(torch.equal(out, ref)) + + def test_matches_softmax_distribution(self): + torch.manual_seed(1) + vocab, n = 8, 200_000 + base = torch.randn(vocab, device="cuda") + logits = base.repeat(n, 1) + correct_len = torch.zeros(n, dtype=torch.int32, device="cuda") + out = sample_simulated_bonus( + target_logits=logits, + correct_len=correct_len, + greedy_bonus=torch.zeros(n, dtype=torch.int64, device="cuda"), + sampling_info=_info([0.7] * n), + bs=n, + verify_num_draft_tokens=1, + ) + freq = torch.bincount(out, minlength=vocab).float() / n + want = torch.softmax(base / 0.7, dim=-1) + self.assertLess((freq - want).abs().max().item(), 5e-3) + + +if __name__ == "__main__": + unittest.main()