[AMD][V4.1][*/N] Greedy dspark draft/accept under SGLANG_SIMULATE_ACC_LEN (#41981)
Co-authored-by: wunhuang <wunhuang@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
wunhuang
Cursor
parent
46637965ab
commit
d669efba84
@@ -0,0 +1,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["seed"])
|
||||
def _simulated_bonus_partial_kernel(
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
temperatures_ptr,
|
||||
part_val_ptr,
|
||||
part_idx_ptr,
|
||||
seed,
|
||||
vocab,
|
||||
ROWS: tl.constexpr,
|
||||
NSPLIT: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
split = tl.program_id(1)
|
||||
t = tl.load(temperatures_ptr + row // ROWS).to(tl.float32)
|
||||
inv_t = 1.0 / tl.maximum(t, 1e-5)
|
||||
offs = split * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < vocab
|
||||
x = tl.load(
|
||||
logits_ptr + row.to(tl.int64) * logits_stride + offs,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
u = tl.rand(seed, row * vocab + offs)
|
||||
v = tl.where(mask, x * inv_t - tl.log(-tl.log(u)), float("-inf"))
|
||||
top = tl.max(v, axis=0)
|
||||
idx = tl.min(tl.where(v == top, offs, vocab), axis=0)
|
||||
tl.store(part_val_ptr + row * NSPLIT + split, top)
|
||||
tl.store(part_idx_ptr + row * NSPLIT + split, idx)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _simulated_bonus_select_kernel(
|
||||
part_val_ptr,
|
||||
part_idx_ptr,
|
||||
correct_len_ptr,
|
||||
out_ptr,
|
||||
vocab,
|
||||
ROWS: tl.constexpr,
|
||||
NSPLIT: tl.constexpr,
|
||||
):
|
||||
req = tl.program_id(0)
|
||||
row = req * ROWS + tl.load(correct_len_ptr + req)
|
||||
s = tl.arange(0, NSPLIT)
|
||||
val = tl.load(part_val_ptr + row * NSPLIT + s)
|
||||
idx = tl.load(part_idx_ptr + row * NSPLIT + s)
|
||||
top = tl.max(val, axis=0)
|
||||
tl.store(
|
||||
out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64)
|
||||
)
|
||||
|
||||
|
||||
def simulated_bonus_sample(
|
||||
*,
|
||||
target_logits: torch.Tensor,
|
||||
correct_len: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
bs: int,
|
||||
rows_per_request: int,
|
||||
) -> torch.Tensor:
|
||||
"""Gumbel-max temperature sample of every verify row; returns, per request, the
|
||||
sample of the row at ``correct_len``. [bs] int64."""
|
||||
assert target_logits.stride(1) == 1
|
||||
device = target_logits.device
|
||||
vocab = target_logits.shape[1]
|
||||
block = 4096
|
||||
nsplit = triton.next_power_of_2(triton.cdiv(vocab, block))
|
||||
rows = bs * rows_per_request
|
||||
part_val = torch.empty((rows, nsplit), dtype=torch.float32, device=device)
|
||||
part_idx = torch.empty((rows, nsplit), dtype=torch.int32, device=device)
|
||||
_simulated_bonus_partial_kernel[(rows, nsplit)](
|
||||
target_logits,
|
||||
target_logits.stride(0),
|
||||
temperatures.view(-1),
|
||||
part_val,
|
||||
part_idx,
|
||||
random.getrandbits(31),
|
||||
vocab,
|
||||
ROWS=rows_per_request,
|
||||
NSPLIT=nsplit,
|
||||
BLOCK=block,
|
||||
num_warps=8,
|
||||
)
|
||||
out = torch.empty((bs,), dtype=torch.int64, device=device)
|
||||
_simulated_bonus_select_kernel[(bs,)](
|
||||
part_val,
|
||||
part_idx,
|
||||
correct_len.contiguous(),
|
||||
out,
|
||||
vocab,
|
||||
ROWS=rows_per_request,
|
||||
NSPLIT=nsplit,
|
||||
)
|
||||
return out
|
||||
@@ -521,6 +521,9 @@ class Envs:
|
||||
SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1)
|
||||
SGLANG_SIMULATE_ACC_METHOD = EnvStr("match-expected")
|
||||
SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr("fixed")
|
||||
# DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests
|
||||
# run the greedy draft/accept path and only the bonus token is temperature-sampled.
|
||||
SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True)
|
||||
SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False)
|
||||
SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False)
|
||||
|
||||
|
||||
@@ -26,6 +26,9 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
build_unified_commit_inject_layout,
|
||||
scatter_compact_to_strided_into,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.simulated_bonus import (
|
||||
simulated_bonus_sample,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode
|
||||
@@ -46,9 +49,10 @@ from sglang.srt.speculative.spec_utils import (
|
||||
SIMULATE_ACC_METHOD,
|
||||
sample_simulated_acc_len,
|
||||
)
|
||||
from sglang.srt.utils import is_npu
|
||||
from sglang.srt.utils import is_hip, is_npu
|
||||
from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
|
||||
# Draft proposal probs feeding rejection sampling; the data layer is the
|
||||
@@ -148,6 +152,7 @@ class TargetVerifyExecutor:
|
||||
layout: Optional[RaggedVerifyLayout],
|
||||
prefix_lens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
simulate_bonus_sampling_info=None,
|
||||
) -> AcceptOuts:
|
||||
"""Produce the per-request accept outcome after target verify.
|
||||
|
||||
@@ -159,25 +164,38 @@ class TargetVerifyExecutor:
|
||||
if folded_accept:
|
||||
return self.verify_epilogue.read_accept(bs)
|
||||
|
||||
simulate = self._simulate_acc_len > 0
|
||||
# Simulated acceptance overwrites correct_len below, so a sampling accept
|
||||
# (draft/target softmax + rejection) would be computed only to be discarded.
|
||||
accept_sampling_info = None if simulate and _is_hip else sampling_info
|
||||
correct_len, bonus, cap_trim_lens = accept_draft_tokens(
|
||||
candidates=verify_ids_2d,
|
||||
target_logits=target_logits,
|
||||
draft_block=draft_block,
|
||||
sampling_info=sampling_info,
|
||||
sampling_info=accept_sampling_info,
|
||||
draft_input=draft_input,
|
||||
gamma=self.gamma,
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
cutoff_layout=layout,
|
||||
fused_argmax=self._target_is_dsv41,
|
||||
)
|
||||
if self._simulate_acc_len > 0:
|
||||
if simulate:
|
||||
correct_len = self._simulated_correct_len(
|
||||
bs=bs, dtype=correct_len.dtype, device=correct_len.device
|
||||
)
|
||||
if simulate_bonus_sampling_info is not None:
|
||||
bonus = sample_simulated_bonus(
|
||||
target_logits=target_logits,
|
||||
correct_len=correct_len,
|
||||
greedy_bonus=bonus,
|
||||
sampling_info=simulate_bonus_sampling_info,
|
||||
bs=bs,
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
)
|
||||
|
||||
site = (
|
||||
SpecTpSyncSite.DSPARK_ACCEPT_GREEDY
|
||||
if sampling_info is None or sampling_info.is_all_greedy
|
||||
if accept_sampling_info is None or accept_sampling_info.is_all_greedy
|
||||
else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE
|
||||
)
|
||||
self._tp_sync.sync(site, correct_len)
|
||||
@@ -811,6 +829,27 @@ class DsparkVerifyEpilogue:
|
||||
)
|
||||
|
||||
|
||||
def sample_simulated_bonus(
|
||||
*,
|
||||
target_logits: torch.Tensor,
|
||||
correct_len: torch.Tensor,
|
||||
greedy_bonus: torch.Tensor,
|
||||
sampling_info,
|
||||
bs: int,
|
||||
verify_num_draft_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
"""Temperature-sample every verify row and return the row at the (simulated)
|
||||
correct_len as the bonus token. Temperature only: callers exclude top-p/top-k/min-p."""
|
||||
bonus = simulated_bonus_sample(
|
||||
target_logits=target_logits,
|
||||
correct_len=correct_len,
|
||||
temperatures=sampling_info.temperatures,
|
||||
bs=bs,
|
||||
rows_per_request=verify_num_draft_tokens,
|
||||
)
|
||||
return bonus.to(greedy_bonus.dtype).view_as(greedy_bonus)
|
||||
|
||||
|
||||
def accept_draft_tokens(
|
||||
*,
|
||||
candidates: torch.Tensor,
|
||||
|
||||
@@ -85,6 +85,7 @@ from sglang.srt.speculative.spec_utils import (
|
||||
from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_cuda_alike,
|
||||
is_hip,
|
||||
is_npu,
|
||||
is_pin_memory_available,
|
||||
)
|
||||
@@ -94,6 +95,7 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
@@ -365,6 +367,11 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
|
||||
self._simulate_acc_len = float(envs.SGLANG_SIMULATE_ACC_LEN.get())
|
||||
self._simulate_acc_greedy = (
|
||||
_is_hip
|
||||
and self._simulate_acc_len > 0
|
||||
and envs.SGLANG_SIMULATE_ACC_GREEDY.get()
|
||||
)
|
||||
if (
|
||||
self._simulate_acc_len > 0
|
||||
and self._simulate_acc_len != 1.0
|
||||
@@ -839,6 +846,21 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
|
||||
sampling_info = batch.sampling_info
|
||||
# Simulated acceptance overrides correct_len, so only the bonus token needs the
|
||||
# request's temperature: draft + accept run greedy, the bonus is still sampled.
|
||||
simulate_bonus_sampling_info = None
|
||||
if (
|
||||
self._simulate_acc_greedy
|
||||
and sampling_info is not None
|
||||
and not sampling_info.is_all_greedy
|
||||
and not batch.has_grammar
|
||||
and verify_logits_adjustments_are_noop(sampling_info)
|
||||
and not sampling_info.need_top_p_sampling
|
||||
and not sampling_info.need_top_k_sampling
|
||||
and not sampling_info.need_min_p_sampling
|
||||
):
|
||||
simulate_bonus_sampling_info = sampling_info
|
||||
sampling_info = None
|
||||
with self._draft_context(), self._observers.segment(InfoSegment.DRAFT):
|
||||
proposal = self._proposer.propose(
|
||||
batch=batch,
|
||||
@@ -963,6 +985,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
layout=layout,
|
||||
prefix_lens=prefix_lens,
|
||||
draft_tokens=draft_tokens,
|
||||
simulate_bonus_sampling_info=simulate_bonus_sampling_info,
|
||||
)
|
||||
self.model_runner.ngram_embedding_manager.update_after_verify(
|
||||
verify_ids_2d=verify_ids_2d,
|
||||
|
||||
Reference in New Issue
Block a user