[Feat] speculative: Seed rejection-sampling draft proposals for deterministic inference (#33395)
Co-authored-by: Ajit Mistry <ajmistry@nvidia.com> Co-authored-by: Ajit Mistry <ajmistry@amistry.nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Ajit Mistry
Ajit Mistry
Po-Han Huang
Po-Han Huang
parent
e5db41b329
commit
25ad569ff9
@@ -1141,6 +1141,12 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
raise ValueError(
|
||||
"--speculative-use-rejection-sampling requires --speculative-eagle-topk=1."
|
||||
)
|
||||
if cfg.enable_deterministic_inference and not cfg.device.startswith("cuda"):
|
||||
raise ValueError(
|
||||
"Deterministic rejection sampling is currently supported only on "
|
||||
"CUDA-compatible devices because draft proposal sampling uses a "
|
||||
"Triton hash kernel."
|
||||
)
|
||||
if (
|
||||
cfg.speculative_accept_threshold_single != 1.0
|
||||
or cfg.speculative_accept_threshold_acc != 1.0
|
||||
@@ -1151,13 +1157,6 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"--speculative-accept-threshold-acc; rejection sampling ignores "
|
||||
"the accept thresholds."
|
||||
)
|
||||
if cfg.enable_deterministic_inference:
|
||||
raise ValueError(
|
||||
"--speculative-use-rejection-sampling is incompatible with "
|
||||
"--enable-deterministic-inference; the sampling kernel draws "
|
||||
"coins from the global RNG and is not batch-invariant."
|
||||
)
|
||||
|
||||
if (
|
||||
resolved_view(server_args).enable_multi_layer_eagle
|
||||
and cfg.speculative_eagle_topk != 1
|
||||
|
||||
@@ -77,6 +77,7 @@ class EagleDraftInputBuffers(ForwardInputBuffers):
|
||||
topk_p: torch.Tensor
|
||||
topk_index: torch.Tensor
|
||||
draft_probs: Optional[torch.Tensor]
|
||||
sampling_seed: Optional[torch.Tensor]
|
||||
hidden_states: Optional[torch.Tensor]
|
||||
global_num_tokens_gpu: Optional[torch.Tensor]
|
||||
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor]
|
||||
@@ -203,6 +204,12 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
if get_spec().speculative_use_rejection_sampling
|
||||
else None
|
||||
)
|
||||
sampling_seed = (
|
||||
torch.zeros((self.max_bs,), dtype=torch.int64)
|
||||
if get_exec().deterministic.enable_deterministic_inference
|
||||
and get_spec().speculative_use_rejection_sampling
|
||||
else None
|
||||
)
|
||||
_hidden_size, _hidden_dtype = get_draft_recurrent_hidden_state_spec(
|
||||
model_runner
|
||||
)
|
||||
@@ -271,6 +278,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
topk_p=topk_p,
|
||||
topk_index=topk_index,
|
||||
draft_probs=draft_probs,
|
||||
sampling_seed=sampling_seed,
|
||||
hidden_states=hidden_states,
|
||||
global_num_tokens_gpu=global_num_tokens_gpu,
|
||||
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
||||
@@ -456,6 +464,11 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
need_top_k_sampling=False,
|
||||
need_min_p_sampling=False,
|
||||
vocab_size=self.model_runner.model_config.vocab_size,
|
||||
sampling_seed=(
|
||||
self.buffers.sampling_seed[:num_seqs]
|
||||
if self.buffers.sampling_seed is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
@@ -671,6 +684,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.temperatures[:raw_bs].copy_(
|
||||
forward_batch.sampling_info.temperatures[:raw_bs]
|
||||
)
|
||||
if self.buffers.sampling_seed is not None:
|
||||
self.buffers.sampling_seed[:raw_bs].copy_(
|
||||
forward_batch.sampling_info.sampling_seed[:raw_bs]
|
||||
)
|
||||
self.top_ks[:raw_bs].copy_(forward_batch.sampling_info.top_ks[:raw_bs])
|
||||
|
||||
# TODO(ch-wan): support num_token_non_padded
|
||||
|
||||
@@ -428,11 +428,9 @@ def _seeded_verify_coins(
|
||||
per-draft rejection coins; column draft_token_num drives the final
|
||||
fallback-sampling coin.
|
||||
|
||||
Scope: this seeds only the verify-side RNG. With rejection sampling the
|
||||
draft workers still pick candidates via unseeded multinomial
|
||||
(fast_sample in eagle_worker_v2), so that mode stays non-deterministic
|
||||
until the draft RNG is seeded in a follow-up; top-k/greedy draft
|
||||
selection is already deterministic.
|
||||
Draft proposal draws use a separate domain-separated seeded stream in
|
||||
``fast_sample``. Keeping the streams separate preserves the independence
|
||||
required by rejection sampling.
|
||||
"""
|
||||
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
||||
|
||||
|
||||
@@ -817,6 +817,9 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
logits_output.next_token_logits,
|
||||
forward_batch.sampling_info.temperatures,
|
||||
forward_batch.sampling_info.top_ks,
|
||||
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||
positions=forward_batch.positions + 1,
|
||||
draft_step=i + 1,
|
||||
)
|
||||
draft_probs_list.append(probs)
|
||||
forward_batch.positions.add_(1)
|
||||
@@ -1031,7 +1034,12 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
use_rejection_sampling,
|
||||
)
|
||||
if use_rejection_sampling:
|
||||
topk_p, topk_index = fast_sample(probs, num_samples=1)
|
||||
topk_p, topk_index = fast_sample(
|
||||
probs,
|
||||
num_samples=1,
|
||||
sampling_seed=batch.sampling_info.sampling_seed,
|
||||
positions=batch.seq_lens,
|
||||
)
|
||||
else:
|
||||
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
|
||||
return EagleDraftInput(
|
||||
@@ -1199,6 +1207,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
draft_logits_output.next_token_logits,
|
||||
batch.sampling_info.temperatures,
|
||||
batch.sampling_info.top_ks,
|
||||
sampling_seed=batch.sampling_info.sampling_seed,
|
||||
positions=batch_result.new_seq_lens,
|
||||
)
|
||||
elif self.topk == 1 and _is_hip:
|
||||
ret_topk_p, ret_topk_index = draft_topk1_argmax_only(
|
||||
|
||||
@@ -136,6 +136,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers):
|
||||
# of draft_probs selects the in-graph proposal branch in _run_step_body).
|
||||
temperatures: Optional[torch.Tensor]
|
||||
draft_probs: Optional[torch.Tensor]
|
||||
sampling_seed: Optional[torch.Tensor]
|
||||
|
||||
|
||||
class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
@@ -394,7 +395,17 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
buffers.temperatures[:bs],
|
||||
buffers.draft_probs[:bs, self.step],
|
||||
)
|
||||
ret.topk_p, ret.topk_index = fast_sample(probs, num_samples=1)
|
||||
ret.topk_p, ret.topk_index = fast_sample(
|
||||
probs,
|
||||
num_samples=1,
|
||||
sampling_seed=(
|
||||
buffers.sampling_seed[:bs]
|
||||
if buffers.sampling_seed is not None
|
||||
else None
|
||||
),
|
||||
positions=buffers.seq_lens[:bs],
|
||||
draft_step=self.step,
|
||||
)
|
||||
|
||||
def _run_step_body(self, forward_batch: ForwardBatch, num_tokens: int, bs: int):
|
||||
"""One draft step's body: model forward + chain-hidden write + top-k.
|
||||
@@ -693,9 +704,15 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
(max_bs, self.speculative_num_steps, vocab_size),
|
||||
dtype=torch.float,
|
||||
)
|
||||
sampling_seed = (
|
||||
torch.zeros((max_bs,), dtype=torch.int64)
|
||||
if get_exec().deterministic.enable_deterministic_inference
|
||||
else None
|
||||
)
|
||||
else:
|
||||
temperatures = None
|
||||
draft_probs = None
|
||||
sampling_seed = None
|
||||
|
||||
if self.require_gathered_buffer:
|
||||
if self.require_mlp_tp_gather:
|
||||
@@ -739,6 +756,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
||||
temperatures=temperatures,
|
||||
draft_probs=draft_probs,
|
||||
sampling_seed=sampling_seed,
|
||||
)
|
||||
|
||||
def _prepare_extra(self, forward_batch: ForwardBatch) -> None:
|
||||
@@ -867,6 +885,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
||||
self.seq_len_fill_value,
|
||||
)
|
||||
|
||||
if buffers.sampling_seed is not None:
|
||||
buffers.sampling_seed[:raw_bs].copy_(
|
||||
forward_batch.sampling_info.sampling_seed[:raw_bs]
|
||||
)
|
||||
|
||||
# Refresh the host mirror only when published; hand replay None
|
||||
# otherwise so no consumer reads a stale buffer.
|
||||
if forward_batch.seq_lens_cpu is not None:
|
||||
@@ -968,10 +991,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
Rejection sampling is supported by sampling X ~ q inside the graph
|
||||
(_sample_draft_proposal, selected by the draft_probs buffer's presence):
|
||||
the captured rotation then carries the sampled token, prepare() stages the
|
||||
temperatures sampling_info cannot deliver in-graph, and the worker clones
|
||||
the per-step q off buffers.draft_probs after replay. torch.multinomial
|
||||
draws through the graph-registered Philox generator, so each replay gets
|
||||
fresh coins.
|
||||
temperatures and sampling seeds cannot be delivered in-graph, and the
|
||||
worker clones the per-step q off buffers.draft_probs after replay. Seeded
|
||||
draws are keyed by request position and draft step; unseeded draws keep
|
||||
using the graph-registered Philox generator.
|
||||
"""
|
||||
|
||||
rotates_in_graph = True
|
||||
|
||||
@@ -705,6 +705,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
probs, topk_p, topk_index = sample_draft_proposal(
|
||||
output.logits_output.next_token_logits,
|
||||
forward_batch.sampling_info.temperatures,
|
||||
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||
positions=forward_batch.seq_lens,
|
||||
draft_step=step,
|
||||
)
|
||||
draft_probs_list.append(probs)
|
||||
else:
|
||||
@@ -912,6 +915,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
||||
step_logits,
|
||||
forward_batch.sampling_info.temperatures,
|
||||
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||
positions=forward_batch.seq_lens,
|
||||
draft_step=step,
|
||||
)
|
||||
ret_draft_probs_list.append(probs)
|
||||
if rotates_in_graph:
|
||||
@@ -985,7 +991,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
]
|
||||
if self.use_rejection_sampling and self.topk == 1:
|
||||
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
||||
logits_sel, forward_batch.sampling_info.temperatures
|
||||
logits_sel,
|
||||
forward_batch.sampling_info.temperatures,
|
||||
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||
positions=forward_batch.seq_lens,
|
||||
draft_step=step,
|
||||
)
|
||||
ret_draft_probs_list.append(probs)
|
||||
else:
|
||||
|
||||
@@ -105,24 +105,93 @@ def resolve_num_tokens_per_req(
|
||||
raise ValueError(f"Unknown speculative phase: {phase}")
|
||||
|
||||
|
||||
def fast_sample(probs: torch.Tensor, num_samples: int = 1):
|
||||
"""Draw from `probs` via the Gumbel-max trick: argmax(probs / Exp(1)).
|
||||
# Domain-separate draft proposal draws from target sampling and verify coins.
|
||||
# The per-step offset also separates successive draft draws if their absolute
|
||||
# position inputs overlap.
|
||||
_DRAFT_GUMBEL_SEED_SALT = 0x44524146545F0000
|
||||
|
||||
Distributionally equivalent to torch.multinomial, but avoids multinomial's
|
||||
device-side distribution-validity assert, which the draft CUDA graph would
|
||||
otherwise capture and replay every step. q is clamped off zero so a zero
|
||||
draw can't yield inf/NaN scores that argmax would wrongly select; fp32
|
||||
avoids bf16 argmax ties biasing the draw. Set SGLANG_OPT_USE_GUMBEL_SAMPLE=0
|
||||
to fall back to torch.multinomial.
|
||||
|
||||
def fast_sample(
|
||||
probs: torch.Tensor,
|
||||
num_samples: int = 1,
|
||||
*,
|
||||
sampling_seed: Optional[torch.Tensor] = None,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
draft_step: int = 0,
|
||||
):
|
||||
"""Draw from ``probs`` with the exponential-race form of Gumbel-max.
|
||||
|
||||
When ``sampling_seed`` is provided, derive one exponential variate per
|
||||
request, position, draft step, and vocabulary entry with ``murmur_hash32``.
|
||||
This makes draft proposals reproducible under batch reordering and CUDA
|
||||
graph replay. The midpoint uint32 mapping stays strictly inside ``(0, 1)``,
|
||||
avoiding infinite exponential/Gumbel values at either hash endpoint.
|
||||
|
||||
The unseeded path keeps using PyTorch's fast RNG-backed implementation.
|
||||
Set ``SGLANG_OPT_USE_GUMBEL_SAMPLE=0`` to make that path fall back to
|
||||
``torch.multinomial``.
|
||||
"""
|
||||
if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get():
|
||||
sample_index = torch.multinomial(probs, num_samples=num_samples)
|
||||
return probs.gather(1, sample_index), sample_index
|
||||
q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0)
|
||||
q.clamp_min_(torch.finfo(torch.float32).tiny)
|
||||
scores = probs.float() / q
|
||||
if sampling_seed is not None:
|
||||
if positions is None:
|
||||
raise ValueError("positions are required for deterministic fast_sample")
|
||||
if sampling_seed.shape != positions.shape:
|
||||
raise ValueError(
|
||||
"sampling_seed and positions must have matching shapes, got "
|
||||
f"{sampling_seed.shape} and {positions.shape}"
|
||||
)
|
||||
if sampling_seed.numel() != probs.shape[0]:
|
||||
raise ValueError(
|
||||
"sampling_seed must have one value per probability row, got "
|
||||
f"{sampling_seed.numel()} seeds for {probs.shape[0]} rows"
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
||||
|
||||
if draft_step < 0:
|
||||
raise ValueError(f"draft_step must be non-negative, got {draft_step}")
|
||||
vocab_size = probs.shape[1]
|
||||
col_offset = draft_step * vocab_size
|
||||
if col_offset + vocab_size > 2**32:
|
||||
raise ValueError(
|
||||
"draft_step and vocabulary size exceed the uint32 hash-key range: "
|
||||
f"draft_step={draft_step}, vocab_size={vocab_size}"
|
||||
)
|
||||
|
||||
# Encode (draft_step, token_id) in the uint32 column block instead of
|
||||
# altering the request seed; step zero retains its established stream.
|
||||
salted_seed = sampling_seed + _DRAFT_GUMBEL_SEED_SALT
|
||||
col_indices = torch.arange(
|
||||
col_offset,
|
||||
col_offset + vocab_size,
|
||||
dtype=torch.int64,
|
||||
device=probs.device,
|
||||
)
|
||||
hashed = murmur_hash32(
|
||||
salted_seed.to(torch.uint64), positions.to(torch.uint64), col_indices
|
||||
)
|
||||
|
||||
# If U ~ Uniform(0, 1), then -log(U) ~ Exp(1), and
|
||||
# argmax_i probs[i] / Exp_i samples from the categorical distribution.
|
||||
# Keep float64 for this first implementation, matching the established
|
||||
# deterministic sampler's numerical-conservatism policy.
|
||||
scores = hashed.to(torch.float64)
|
||||
del hashed
|
||||
scores.add_(0.5).div_(2.0**32).log_().neg_().reciprocal_()
|
||||
scores.mul_(probs)
|
||||
else:
|
||||
if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get():
|
||||
sample_index = torch.multinomial(probs, num_samples=num_samples)
|
||||
return probs.gather(1, sample_index), sample_index
|
||||
q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0)
|
||||
q.clamp_min_(torch.finfo(torch.float32).tiny)
|
||||
scores = probs.float() / q
|
||||
|
||||
if num_samples == 1:
|
||||
sample_index = row_argmax(scores).unsqueeze(-1)
|
||||
# row_argmax is FP32-only; the seeded path keeps float64 scores.
|
||||
if scores.dtype == torch.float32:
|
||||
sample_index = row_argmax(scores).unsqueeze(-1)
|
||||
else:
|
||||
sample_index = scores.argmax(dim=-1, keepdim=True)
|
||||
else:
|
||||
sample_index = scores.topk(num_samples, dim=-1).indices
|
||||
sample_p = probs.gather(1, sample_index)
|
||||
@@ -149,6 +218,10 @@ def sample_draft_proposal(
|
||||
next_token_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
top_ks: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
sampling_seed: Optional[torch.Tensor] = None,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
draft_step: int = 0,
|
||||
):
|
||||
"""Leviathan draft proposal: q = softmax(logits / T), X ~ q.
|
||||
|
||||
@@ -171,7 +244,13 @@ def sample_draft_proposal(
|
||||
which is what greedy means. Drop that renorm and this stops holding.
|
||||
"""
|
||||
probs = sampling_softmax(next_token_logits, temperatures=temperatures)
|
||||
topk_p, topk_index = fast_sample(probs, num_samples=1)
|
||||
topk_p, topk_index = fast_sample(
|
||||
probs,
|
||||
num_samples=1,
|
||||
sampling_seed=sampling_seed,
|
||||
positions=positions,
|
||||
draft_step=draft_step,
|
||||
)
|
||||
if top_ks is not None:
|
||||
# Assert rather than skip on a device mismatch: a host-side top_ks would
|
||||
# make this correction silently vanish, and the symptom -- draft accept
|
||||
|
||||
Reference in New Issue
Block a user