[AMD][V4.1][*/N] Greedy dspark draft/accept under SGLANG_SIMULATE_ACC_LEN (#41981)

Co-authored-by: wunhuang <wunhuang@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Thomas Wang
2026-10-01 03:35:28 -07:00
committed by GitHub
co-authored by wunhuang Cursor
parent 46637965ab
commit d669efba84
5 changed files with 252 additions and 4 deletions
@@ -0,0 +1,104 @@
from __future__ import annotations
import random
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["seed"])
def _simulated_bonus_partial_kernel(
logits_ptr,
logits_stride,
temperatures_ptr,
part_val_ptr,
part_idx_ptr,
seed,
vocab,
ROWS: tl.constexpr,
NSPLIT: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
split = tl.program_id(1)
t = tl.load(temperatures_ptr + row // ROWS).to(tl.float32)
inv_t = 1.0 / tl.maximum(t, 1e-5)
offs = split * BLOCK + tl.arange(0, BLOCK)
mask = offs < vocab
x = tl.load(
logits_ptr + row.to(tl.int64) * logits_stride + offs,
mask=mask,
other=float("-inf"),
).to(tl.float32)
u = tl.rand(seed, row * vocab + offs)
v = tl.where(mask, x * inv_t - tl.log(-tl.log(u)), float("-inf"))
top = tl.max(v, axis=0)
idx = tl.min(tl.where(v == top, offs, vocab), axis=0)
tl.store(part_val_ptr + row * NSPLIT + split, top)
tl.store(part_idx_ptr + row * NSPLIT + split, idx)
@triton.jit
def _simulated_bonus_select_kernel(
part_val_ptr,
part_idx_ptr,
correct_len_ptr,
out_ptr,
vocab,
ROWS: tl.constexpr,
NSPLIT: tl.constexpr,
):
req = tl.program_id(0)
row = req * ROWS + tl.load(correct_len_ptr + req)
s = tl.arange(0, NSPLIT)
val = tl.load(part_val_ptr + row * NSPLIT + s)
idx = tl.load(part_idx_ptr + row * NSPLIT + s)
top = tl.max(val, axis=0)
tl.store(
out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64)
)
def simulated_bonus_sample(
*,
target_logits: torch.Tensor,
correct_len: torch.Tensor,
temperatures: torch.Tensor,
bs: int,
rows_per_request: int,
) -> torch.Tensor:
"""Gumbel-max temperature sample of every verify row; returns, per request, the
sample of the row at ``correct_len``. [bs] int64."""
assert target_logits.stride(1) == 1
device = target_logits.device
vocab = target_logits.shape[1]
block = 4096
nsplit = triton.next_power_of_2(triton.cdiv(vocab, block))
rows = bs * rows_per_request
part_val = torch.empty((rows, nsplit), dtype=torch.float32, device=device)
part_idx = torch.empty((rows, nsplit), dtype=torch.int32, device=device)
_simulated_bonus_partial_kernel[(rows, nsplit)](
target_logits,
target_logits.stride(0),
temperatures.view(-1),
part_val,
part_idx,
random.getrandbits(31),
vocab,
ROWS=rows_per_request,
NSPLIT=nsplit,
BLOCK=block,
num_warps=8,
)
out = torch.empty((bs,), dtype=torch.int64, device=device)
_simulated_bonus_select_kernel[(bs,)](
part_val,
part_idx,
correct_len.contiguous(),
out,
vocab,
ROWS=rows_per_request,
NSPLIT=nsplit,
)
return out
+3
View File
@@ -521,6 +521,9 @@ class Envs:
SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1)
SGLANG_SIMULATE_ACC_METHOD = EnvStr("match-expected")
SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr("fixed")
# DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests
# run the greedy draft/accept path and only the bonus token is temperature-sampled.
SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True)
SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False)
SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False)
@@ -26,6 +26,9 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
build_unified_commit_inject_layout,
scatter_compact_to_strided_into,
)
from sglang.kernels.ops.speculative.dspark.simulated_bonus import (
simulated_bonus_sample,
)
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode
@@ -46,9 +49,10 @@ from sglang.srt.speculative.spec_utils import (
SIMULATE_ACC_METHOD,
sample_simulated_acc_len,
)
from sglang.srt.utils import is_npu
from sglang.srt.utils import is_hip, is_npu
from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect
_is_hip = is_hip()
_is_npu = is_npu()
# Draft proposal probs feeding rejection sampling; the data layer is the
@@ -148,6 +152,7 @@ class TargetVerifyExecutor:
layout: Optional[RaggedVerifyLayout],
prefix_lens: torch.Tensor,
draft_tokens: torch.Tensor,
simulate_bonus_sampling_info=None,
) -> AcceptOuts:
"""Produce the per-request accept outcome after target verify.
@@ -159,25 +164,38 @@ class TargetVerifyExecutor:
if folded_accept:
return self.verify_epilogue.read_accept(bs)
simulate = self._simulate_acc_len > 0
# Simulated acceptance overwrites correct_len below, so a sampling accept
# (draft/target softmax + rejection) would be computed only to be discarded.
accept_sampling_info = None if simulate and _is_hip else sampling_info
correct_len, bonus, cap_trim_lens = accept_draft_tokens(
candidates=verify_ids_2d,
target_logits=target_logits,
draft_block=draft_block,
sampling_info=sampling_info,
sampling_info=accept_sampling_info,
draft_input=draft_input,
gamma=self.gamma,
verify_num_draft_tokens=self.verify_num_draft_tokens,
cutoff_layout=layout,
fused_argmax=self._target_is_dsv41,
)
if self._simulate_acc_len > 0:
if simulate:
correct_len = self._simulated_correct_len(
bs=bs, dtype=correct_len.dtype, device=correct_len.device
)
if simulate_bonus_sampling_info is not None:
bonus = sample_simulated_bonus(
target_logits=target_logits,
correct_len=correct_len,
greedy_bonus=bonus,
sampling_info=simulate_bonus_sampling_info,
bs=bs,
verify_num_draft_tokens=self.verify_num_draft_tokens,
)
site = (
SpecTpSyncSite.DSPARK_ACCEPT_GREEDY
if sampling_info is None or sampling_info.is_all_greedy
if accept_sampling_info is None or accept_sampling_info.is_all_greedy
else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE
)
self._tp_sync.sync(site, correct_len)
@@ -811,6 +829,27 @@ class DsparkVerifyEpilogue:
)
def sample_simulated_bonus(
*,
target_logits: torch.Tensor,
correct_len: torch.Tensor,
greedy_bonus: torch.Tensor,
sampling_info,
bs: int,
verify_num_draft_tokens: int,
) -> torch.Tensor:
"""Temperature-sample every verify row and return the row at the (simulated)
correct_len as the bonus token. Temperature only: callers exclude top-p/top-k/min-p."""
bonus = simulated_bonus_sample(
target_logits=target_logits,
correct_len=correct_len,
temperatures=sampling_info.temperatures,
bs=bs,
rows_per_request=verify_num_draft_tokens,
)
return bonus.to(greedy_bonus.dtype).view_as(greedy_bonus)
def accept_draft_tokens(
*,
candidates: torch.Tensor,
@@ -85,6 +85,7 @@ from sglang.srt.speculative.spec_utils import (
from sglang.srt.utils import (
is_cuda,
is_cuda_alike,
is_hip,
is_npu,
is_pin_memory_available,
)
@@ -94,6 +95,7 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_is_hip = is_hip()
_is_npu = is_npu()
@@ -365,6 +367,11 @@ class DSparkWorkerV2(BaseSpecWorker):
)
self._simulate_acc_len = float(envs.SGLANG_SIMULATE_ACC_LEN.get())
self._simulate_acc_greedy = (
_is_hip
and self._simulate_acc_len > 0
and envs.SGLANG_SIMULATE_ACC_GREEDY.get()
)
if (
self._simulate_acc_len > 0
and self._simulate_acc_len != 1.0
@@ -839,6 +846,21 @@ class DSparkWorkerV2(BaseSpecWorker):
)
sampling_info = batch.sampling_info
# Simulated acceptance overrides correct_len, so only the bonus token needs the
# request's temperature: draft + accept run greedy, the bonus is still sampled.
simulate_bonus_sampling_info = None
if (
self._simulate_acc_greedy
and sampling_info is not None
and not sampling_info.is_all_greedy
and not batch.has_grammar
and verify_logits_adjustments_are_noop(sampling_info)
and not sampling_info.need_top_p_sampling
and not sampling_info.need_top_k_sampling
and not sampling_info.need_min_p_sampling
):
simulate_bonus_sampling_info = sampling_info
sampling_info = None
with self._draft_context(), self._observers.segment(InfoSegment.DRAFT):
proposal = self._proposer.propose(
batch=batch,
@@ -963,6 +985,7 @@ class DSparkWorkerV2(BaseSpecWorker):
layout=layout,
prefix_lens=prefix_lens,
draft_tokens=draft_tokens,
simulate_bonus_sampling_info=simulate_bonus_sampling_info,
)
self.model_runner.ngram_embedding_manager.update_after_verify(
verify_ids_2d=verify_ids_2d,