[Feat] speculative: Seed rejection-sampling draft proposals for deterministic inference (#33395)
Co-authored-by: Ajit Mistry <ajmistry@nvidia.com> Co-authored-by: Ajit Mistry <ajmistry@amistry.nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Ajit Mistry
Ajit Mistry
Po-Han Huang
Po-Han Huang
parent
e5db41b329
commit
25ad569ff9
@@ -0,0 +1,89 @@
|
|||||||
|
"""Benchmark seeded versus unseeded EAGLE draft Gumbel sampling.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
python benchmark/kernels/sampling/bench_deterministic_fast_sample.py \
|
||||||
|
--batch-sizes 1 8 32 128 --vocab-sizes 32000 128256 163840
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
import torch
|
||||||
|
import triton
|
||||||
|
|
||||||
|
from sglang.srt.speculative.spec_utils import fast_sample
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--batch-sizes", type=int, nargs="+", default=[1, 8, 32, 128])
|
||||||
|
parser.add_argument(
|
||||||
|
"--vocab-sizes", type=int, nargs="+", default=[32000, 128256, 163840]
|
||||||
|
)
|
||||||
|
parser.add_argument("--warmup-ms", type=int, default=100)
|
||||||
|
parser.add_argument("--rep-ms", type=int, default=500)
|
||||||
|
parser.add_argument(
|
||||||
|
"--modes", nargs="+", choices=("eager", "graph"), default=["graph"]
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def capture(fn):
|
||||||
|
fn()
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(graph):
|
||||||
|
fn()
|
||||||
|
return graph.replay
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
device = torch.device("cuda")
|
||||||
|
print(f"device={torch.cuda.get_device_name(device)}")
|
||||||
|
print("mode\tbatch\tvocab\tunseeded_us\tseeded_us\tslowdown")
|
||||||
|
|
||||||
|
for vocab_size in args.vocab_sizes:
|
||||||
|
for batch_size in args.batch_sizes:
|
||||||
|
logits = torch.randn(
|
||||||
|
(batch_size, vocab_size), device=device, dtype=torch.float32
|
||||||
|
)
|
||||||
|
probs = torch.softmax(logits, dim=-1)
|
||||||
|
seeds = torch.arange(batch_size, device=device, dtype=torch.int64) + 12345
|
||||||
|
positions = (
|
||||||
|
torch.arange(batch_size, device=device, dtype=torch.int64) + 4096
|
||||||
|
)
|
||||||
|
|
||||||
|
def unseeded():
|
||||||
|
return fast_sample(probs)
|
||||||
|
|
||||||
|
def seeded():
|
||||||
|
return fast_sample(
|
||||||
|
probs,
|
||||||
|
sampling_seed=seeds,
|
||||||
|
positions=positions,
|
||||||
|
draft_step=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Compile/JIT and populate allocator caches before timing.
|
||||||
|
unseeded()
|
||||||
|
seeded()
|
||||||
|
torch.cuda.synchronize()
|
||||||
|
|
||||||
|
for mode in args.modes:
|
||||||
|
unseeded_fn = capture(unseeded) if mode == "graph" else unseeded
|
||||||
|
seeded_fn = capture(seeded) if mode == "graph" else seeded
|
||||||
|
unseeded_ms = triton.testing.do_bench(
|
||||||
|
unseeded_fn, warmup=args.warmup_ms, rep=args.rep_ms
|
||||||
|
)
|
||||||
|
seeded_ms = triton.testing.do_bench(
|
||||||
|
seeded_fn, warmup=args.warmup_ms, rep=args.rep_ms
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"{mode}\t{batch_size}\t{vocab_size}\t"
|
||||||
|
f"{unseeded_ms * 1000:.1f}\t{seeded_ms * 1000:.1f}\t"
|
||||||
|
f"{seeded_ms / unseeded_ms:.2f}x"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1141,6 +1141,12 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
"--speculative-use-rejection-sampling requires --speculative-eagle-topk=1."
|
"--speculative-use-rejection-sampling requires --speculative-eagle-topk=1."
|
||||||
)
|
)
|
||||||
|
if cfg.enable_deterministic_inference and not cfg.device.startswith("cuda"):
|
||||||
|
raise ValueError(
|
||||||
|
"Deterministic rejection sampling is currently supported only on "
|
||||||
|
"CUDA-compatible devices because draft proposal sampling uses a "
|
||||||
|
"Triton hash kernel."
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
cfg.speculative_accept_threshold_single != 1.0
|
cfg.speculative_accept_threshold_single != 1.0
|
||||||
or cfg.speculative_accept_threshold_acc != 1.0
|
or cfg.speculative_accept_threshold_acc != 1.0
|
||||||
@@ -1151,13 +1157,6 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
|||||||
"--speculative-accept-threshold-acc; rejection sampling ignores "
|
"--speculative-accept-threshold-acc; rejection sampling ignores "
|
||||||
"the accept thresholds."
|
"the accept thresholds."
|
||||||
)
|
)
|
||||||
if cfg.enable_deterministic_inference:
|
|
||||||
raise ValueError(
|
|
||||||
"--speculative-use-rejection-sampling is incompatible with "
|
|
||||||
"--enable-deterministic-inference; the sampling kernel draws "
|
|
||||||
"coins from the global RNG and is not batch-invariant."
|
|
||||||
)
|
|
||||||
|
|
||||||
if (
|
if (
|
||||||
resolved_view(server_args).enable_multi_layer_eagle
|
resolved_view(server_args).enable_multi_layer_eagle
|
||||||
and cfg.speculative_eagle_topk != 1
|
and cfg.speculative_eagle_topk != 1
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ class EagleDraftInputBuffers(ForwardInputBuffers):
|
|||||||
topk_p: torch.Tensor
|
topk_p: torch.Tensor
|
||||||
topk_index: torch.Tensor
|
topk_index: torch.Tensor
|
||||||
draft_probs: Optional[torch.Tensor]
|
draft_probs: Optional[torch.Tensor]
|
||||||
|
sampling_seed: Optional[torch.Tensor]
|
||||||
hidden_states: Optional[torch.Tensor]
|
hidden_states: Optional[torch.Tensor]
|
||||||
global_num_tokens_gpu: Optional[torch.Tensor]
|
global_num_tokens_gpu: Optional[torch.Tensor]
|
||||||
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor]
|
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor]
|
||||||
@@ -203,6 +204,12 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
|||||||
if get_spec().speculative_use_rejection_sampling
|
if get_spec().speculative_use_rejection_sampling
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
sampling_seed = (
|
||||||
|
torch.zeros((self.max_bs,), dtype=torch.int64)
|
||||||
|
if get_exec().deterministic.enable_deterministic_inference
|
||||||
|
and get_spec().speculative_use_rejection_sampling
|
||||||
|
else None
|
||||||
|
)
|
||||||
_hidden_size, _hidden_dtype = get_draft_recurrent_hidden_state_spec(
|
_hidden_size, _hidden_dtype = get_draft_recurrent_hidden_state_spec(
|
||||||
model_runner
|
model_runner
|
||||||
)
|
)
|
||||||
@@ -271,6 +278,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
|||||||
topk_p=topk_p,
|
topk_p=topk_p,
|
||||||
topk_index=topk_index,
|
topk_index=topk_index,
|
||||||
draft_probs=draft_probs,
|
draft_probs=draft_probs,
|
||||||
|
sampling_seed=sampling_seed,
|
||||||
hidden_states=hidden_states,
|
hidden_states=hidden_states,
|
||||||
global_num_tokens_gpu=global_num_tokens_gpu,
|
global_num_tokens_gpu=global_num_tokens_gpu,
|
||||||
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
||||||
@@ -456,6 +464,11 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
|||||||
need_top_k_sampling=False,
|
need_top_k_sampling=False,
|
||||||
need_min_p_sampling=False,
|
need_min_p_sampling=False,
|
||||||
vocab_size=self.model_runner.model_config.vocab_size,
|
vocab_size=self.model_runner.model_config.vocab_size,
|
||||||
|
sampling_seed=(
|
||||||
|
self.buffers.sampling_seed[:num_seqs]
|
||||||
|
if self.buffers.sampling_seed is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
forward_batch = ForwardBatch(
|
forward_batch = ForwardBatch(
|
||||||
@@ -671,6 +684,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
|||||||
self.temperatures[:raw_bs].copy_(
|
self.temperatures[:raw_bs].copy_(
|
||||||
forward_batch.sampling_info.temperatures[:raw_bs]
|
forward_batch.sampling_info.temperatures[:raw_bs]
|
||||||
)
|
)
|
||||||
|
if self.buffers.sampling_seed is not None:
|
||||||
|
self.buffers.sampling_seed[:raw_bs].copy_(
|
||||||
|
forward_batch.sampling_info.sampling_seed[:raw_bs]
|
||||||
|
)
|
||||||
self.top_ks[:raw_bs].copy_(forward_batch.sampling_info.top_ks[:raw_bs])
|
self.top_ks[:raw_bs].copy_(forward_batch.sampling_info.top_ks[:raw_bs])
|
||||||
|
|
||||||
# TODO(ch-wan): support num_token_non_padded
|
# TODO(ch-wan): support num_token_non_padded
|
||||||
|
|||||||
@@ -428,11 +428,9 @@ def _seeded_verify_coins(
|
|||||||
per-draft rejection coins; column draft_token_num drives the final
|
per-draft rejection coins; column draft_token_num drives the final
|
||||||
fallback-sampling coin.
|
fallback-sampling coin.
|
||||||
|
|
||||||
Scope: this seeds only the verify-side RNG. With rejection sampling the
|
Draft proposal draws use a separate domain-separated seeded stream in
|
||||||
draft workers still pick candidates via unseeded multinomial
|
``fast_sample``. Keeping the streams separate preserves the independence
|
||||||
(fast_sample in eagle_worker_v2), so that mode stays non-deterministic
|
required by rejection sampling.
|
||||||
until the draft RNG is seeded in a follow-up; top-k/greedy draft
|
|
||||||
selection is already deterministic.
|
|
||||||
"""
|
"""
|
||||||
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
||||||
|
|
||||||
|
|||||||
@@ -817,6 +817,9 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
logits_output.next_token_logits,
|
logits_output.next_token_logits,
|
||||||
forward_batch.sampling_info.temperatures,
|
forward_batch.sampling_info.temperatures,
|
||||||
forward_batch.sampling_info.top_ks,
|
forward_batch.sampling_info.top_ks,
|
||||||
|
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||||
|
positions=forward_batch.positions + 1,
|
||||||
|
draft_step=i + 1,
|
||||||
)
|
)
|
||||||
draft_probs_list.append(probs)
|
draft_probs_list.append(probs)
|
||||||
forward_batch.positions.add_(1)
|
forward_batch.positions.add_(1)
|
||||||
@@ -1031,7 +1034,12 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
use_rejection_sampling,
|
use_rejection_sampling,
|
||||||
)
|
)
|
||||||
if use_rejection_sampling:
|
if use_rejection_sampling:
|
||||||
topk_p, topk_index = fast_sample(probs, num_samples=1)
|
topk_p, topk_index = fast_sample(
|
||||||
|
probs,
|
||||||
|
num_samples=1,
|
||||||
|
sampling_seed=batch.sampling_info.sampling_seed,
|
||||||
|
positions=batch.seq_lens,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
|
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
|
||||||
return EagleDraftInput(
|
return EagleDraftInput(
|
||||||
@@ -1199,6 +1207,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
draft_logits_output.next_token_logits,
|
draft_logits_output.next_token_logits,
|
||||||
batch.sampling_info.temperatures,
|
batch.sampling_info.temperatures,
|
||||||
batch.sampling_info.top_ks,
|
batch.sampling_info.top_ks,
|
||||||
|
sampling_seed=batch.sampling_info.sampling_seed,
|
||||||
|
positions=batch_result.new_seq_lens,
|
||||||
)
|
)
|
||||||
elif self.topk == 1 and _is_hip:
|
elif self.topk == 1 and _is_hip:
|
||||||
ret_topk_p, ret_topk_index = draft_topk1_argmax_only(
|
ret_topk_p, ret_topk_index = draft_topk1_argmax_only(
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers):
|
|||||||
# of draft_probs selects the in-graph proposal branch in _run_step_body).
|
# of draft_probs selects the in-graph proposal branch in _run_step_body).
|
||||||
temperatures: Optional[torch.Tensor]
|
temperatures: Optional[torch.Tensor]
|
||||||
draft_probs: Optional[torch.Tensor]
|
draft_probs: Optional[torch.Tensor]
|
||||||
|
sampling_seed: Optional[torch.Tensor]
|
||||||
|
|
||||||
|
|
||||||
class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||||
@@ -394,7 +395,17 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
|||||||
buffers.temperatures[:bs],
|
buffers.temperatures[:bs],
|
||||||
buffers.draft_probs[:bs, self.step],
|
buffers.draft_probs[:bs, self.step],
|
||||||
)
|
)
|
||||||
ret.topk_p, ret.topk_index = fast_sample(probs, num_samples=1)
|
ret.topk_p, ret.topk_index = fast_sample(
|
||||||
|
probs,
|
||||||
|
num_samples=1,
|
||||||
|
sampling_seed=(
|
||||||
|
buffers.sampling_seed[:bs]
|
||||||
|
if buffers.sampling_seed is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
positions=buffers.seq_lens[:bs],
|
||||||
|
draft_step=self.step,
|
||||||
|
)
|
||||||
|
|
||||||
def _run_step_body(self, forward_batch: ForwardBatch, num_tokens: int, bs: int):
|
def _run_step_body(self, forward_batch: ForwardBatch, num_tokens: int, bs: int):
|
||||||
"""One draft step's body: model forward + chain-hidden write + top-k.
|
"""One draft step's body: model forward + chain-hidden write + top-k.
|
||||||
@@ -693,9 +704,15 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
|||||||
(max_bs, self.speculative_num_steps, vocab_size),
|
(max_bs, self.speculative_num_steps, vocab_size),
|
||||||
dtype=torch.float,
|
dtype=torch.float,
|
||||||
)
|
)
|
||||||
|
sampling_seed = (
|
||||||
|
torch.zeros((max_bs,), dtype=torch.int64)
|
||||||
|
if get_exec().deterministic.enable_deterministic_inference
|
||||||
|
else None
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
temperatures = None
|
temperatures = None
|
||||||
draft_probs = None
|
draft_probs = None
|
||||||
|
sampling_seed = None
|
||||||
|
|
||||||
if self.require_gathered_buffer:
|
if self.require_gathered_buffer:
|
||||||
if self.require_mlp_tp_gather:
|
if self.require_mlp_tp_gather:
|
||||||
@@ -739,6 +756,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
|||||||
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu,
|
||||||
temperatures=temperatures,
|
temperatures=temperatures,
|
||||||
draft_probs=draft_probs,
|
draft_probs=draft_probs,
|
||||||
|
sampling_seed=sampling_seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _prepare_extra(self, forward_batch: ForwardBatch) -> None:
|
def _prepare_extra(self, forward_batch: ForwardBatch) -> None:
|
||||||
@@ -867,6 +885,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
|
|||||||
self.seq_len_fill_value,
|
self.seq_len_fill_value,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if buffers.sampling_seed is not None:
|
||||||
|
buffers.sampling_seed[:raw_bs].copy_(
|
||||||
|
forward_batch.sampling_info.sampling_seed[:raw_bs]
|
||||||
|
)
|
||||||
|
|
||||||
# Refresh the host mirror only when published; hand replay None
|
# Refresh the host mirror only when published; hand replay None
|
||||||
# otherwise so no consumer reads a stale buffer.
|
# otherwise so no consumer reads a stale buffer.
|
||||||
if forward_batch.seq_lens_cpu is not None:
|
if forward_batch.seq_lens_cpu is not None:
|
||||||
@@ -968,10 +991,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
|||||||
Rejection sampling is supported by sampling X ~ q inside the graph
|
Rejection sampling is supported by sampling X ~ q inside the graph
|
||||||
(_sample_draft_proposal, selected by the draft_probs buffer's presence):
|
(_sample_draft_proposal, selected by the draft_probs buffer's presence):
|
||||||
the captured rotation then carries the sampled token, prepare() stages the
|
the captured rotation then carries the sampled token, prepare() stages the
|
||||||
temperatures sampling_info cannot deliver in-graph, and the worker clones
|
temperatures and sampling seeds cannot be delivered in-graph, and the
|
||||||
the per-step q off buffers.draft_probs after replay. torch.multinomial
|
worker clones the per-step q off buffers.draft_probs after replay. Seeded
|
||||||
draws through the graph-registered Philox generator, so each replay gets
|
draws are keyed by request position and draft step; unseeded draws keep
|
||||||
fresh coins.
|
using the graph-registered Philox generator.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
rotates_in_graph = True
|
rotates_in_graph = True
|
||||||
|
|||||||
@@ -705,6 +705,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
probs, topk_p, topk_index = sample_draft_proposal(
|
probs, topk_p, topk_index = sample_draft_proposal(
|
||||||
output.logits_output.next_token_logits,
|
output.logits_output.next_token_logits,
|
||||||
forward_batch.sampling_info.temperatures,
|
forward_batch.sampling_info.temperatures,
|
||||||
|
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||||
|
positions=forward_batch.seq_lens,
|
||||||
|
draft_step=step,
|
||||||
)
|
)
|
||||||
draft_probs_list.append(probs)
|
draft_probs_list.append(probs)
|
||||||
else:
|
else:
|
||||||
@@ -912,6 +915,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
||||||
step_logits,
|
step_logits,
|
||||||
forward_batch.sampling_info.temperatures,
|
forward_batch.sampling_info.temperatures,
|
||||||
|
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||||
|
positions=forward_batch.seq_lens,
|
||||||
|
draft_step=step,
|
||||||
)
|
)
|
||||||
ret_draft_probs_list.append(probs)
|
ret_draft_probs_list.append(probs)
|
||||||
if rotates_in_graph:
|
if rotates_in_graph:
|
||||||
@@ -985,7 +991,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
]
|
]
|
||||||
if self.use_rejection_sampling and self.topk == 1:
|
if self.use_rejection_sampling and self.topk == 1:
|
||||||
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
probs, ret_topk_p, ret_topk_index = sample_draft_proposal(
|
||||||
logits_sel, forward_batch.sampling_info.temperatures
|
logits_sel,
|
||||||
|
forward_batch.sampling_info.temperatures,
|
||||||
|
sampling_seed=forward_batch.sampling_info.sampling_seed,
|
||||||
|
positions=forward_batch.seq_lens,
|
||||||
|
draft_step=step,
|
||||||
)
|
)
|
||||||
ret_draft_probs_list.append(probs)
|
ret_draft_probs_list.append(probs)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -105,24 +105,93 @@ def resolve_num_tokens_per_req(
|
|||||||
raise ValueError(f"Unknown speculative phase: {phase}")
|
raise ValueError(f"Unknown speculative phase: {phase}")
|
||||||
|
|
||||||
|
|
||||||
def fast_sample(probs: torch.Tensor, num_samples: int = 1):
|
# Domain-separate draft proposal draws from target sampling and verify coins.
|
||||||
"""Draw from `probs` via the Gumbel-max trick: argmax(probs / Exp(1)).
|
# The per-step offset also separates successive draft draws if their absolute
|
||||||
|
# position inputs overlap.
|
||||||
|
_DRAFT_GUMBEL_SEED_SALT = 0x44524146545F0000
|
||||||
|
|
||||||
Distributionally equivalent to torch.multinomial, but avoids multinomial's
|
|
||||||
device-side distribution-validity assert, which the draft CUDA graph would
|
def fast_sample(
|
||||||
otherwise capture and replay every step. q is clamped off zero so a zero
|
probs: torch.Tensor,
|
||||||
draw can't yield inf/NaN scores that argmax would wrongly select; fp32
|
num_samples: int = 1,
|
||||||
avoids bf16 argmax ties biasing the draw. Set SGLANG_OPT_USE_GUMBEL_SAMPLE=0
|
*,
|
||||||
to fall back to torch.multinomial.
|
sampling_seed: Optional[torch.Tensor] = None,
|
||||||
|
positions: Optional[torch.Tensor] = None,
|
||||||
|
draft_step: int = 0,
|
||||||
|
):
|
||||||
|
"""Draw from ``probs`` with the exponential-race form of Gumbel-max.
|
||||||
|
|
||||||
|
When ``sampling_seed`` is provided, derive one exponential variate per
|
||||||
|
request, position, draft step, and vocabulary entry with ``murmur_hash32``.
|
||||||
|
This makes draft proposals reproducible under batch reordering and CUDA
|
||||||
|
graph replay. The midpoint uint32 mapping stays strictly inside ``(0, 1)``,
|
||||||
|
avoiding infinite exponential/Gumbel values at either hash endpoint.
|
||||||
|
|
||||||
|
The unseeded path keeps using PyTorch's fast RNG-backed implementation.
|
||||||
|
Set ``SGLANG_OPT_USE_GUMBEL_SAMPLE=0`` to make that path fall back to
|
||||||
|
``torch.multinomial``.
|
||||||
"""
|
"""
|
||||||
if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get():
|
if sampling_seed is not None:
|
||||||
sample_index = torch.multinomial(probs, num_samples=num_samples)
|
if positions is None:
|
||||||
return probs.gather(1, sample_index), sample_index
|
raise ValueError("positions are required for deterministic fast_sample")
|
||||||
q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0)
|
if sampling_seed.shape != positions.shape:
|
||||||
q.clamp_min_(torch.finfo(torch.float32).tiny)
|
raise ValueError(
|
||||||
scores = probs.float() / q
|
"sampling_seed and positions must have matching shapes, got "
|
||||||
|
f"{sampling_seed.shape} and {positions.shape}"
|
||||||
|
)
|
||||||
|
if sampling_seed.numel() != probs.shape[0]:
|
||||||
|
raise ValueError(
|
||||||
|
"sampling_seed must have one value per probability row, got "
|
||||||
|
f"{sampling_seed.numel()} seeds for {probs.shape[0]} rows"
|
||||||
|
)
|
||||||
|
|
||||||
|
from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32
|
||||||
|
|
||||||
|
if draft_step < 0:
|
||||||
|
raise ValueError(f"draft_step must be non-negative, got {draft_step}")
|
||||||
|
vocab_size = probs.shape[1]
|
||||||
|
col_offset = draft_step * vocab_size
|
||||||
|
if col_offset + vocab_size > 2**32:
|
||||||
|
raise ValueError(
|
||||||
|
"draft_step and vocabulary size exceed the uint32 hash-key range: "
|
||||||
|
f"draft_step={draft_step}, vocab_size={vocab_size}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Encode (draft_step, token_id) in the uint32 column block instead of
|
||||||
|
# altering the request seed; step zero retains its established stream.
|
||||||
|
salted_seed = sampling_seed + _DRAFT_GUMBEL_SEED_SALT
|
||||||
|
col_indices = torch.arange(
|
||||||
|
col_offset,
|
||||||
|
col_offset + vocab_size,
|
||||||
|
dtype=torch.int64,
|
||||||
|
device=probs.device,
|
||||||
|
)
|
||||||
|
hashed = murmur_hash32(
|
||||||
|
salted_seed.to(torch.uint64), positions.to(torch.uint64), col_indices
|
||||||
|
)
|
||||||
|
|
||||||
|
# If U ~ Uniform(0, 1), then -log(U) ~ Exp(1), and
|
||||||
|
# argmax_i probs[i] / Exp_i samples from the categorical distribution.
|
||||||
|
# Keep float64 for this first implementation, matching the established
|
||||||
|
# deterministic sampler's numerical-conservatism policy.
|
||||||
|
scores = hashed.to(torch.float64)
|
||||||
|
del hashed
|
||||||
|
scores.add_(0.5).div_(2.0**32).log_().neg_().reciprocal_()
|
||||||
|
scores.mul_(probs)
|
||||||
|
else:
|
||||||
|
if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get():
|
||||||
|
sample_index = torch.multinomial(probs, num_samples=num_samples)
|
||||||
|
return probs.gather(1, sample_index), sample_index
|
||||||
|
q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0)
|
||||||
|
q.clamp_min_(torch.finfo(torch.float32).tiny)
|
||||||
|
scores = probs.float() / q
|
||||||
|
|
||||||
if num_samples == 1:
|
if num_samples == 1:
|
||||||
sample_index = row_argmax(scores).unsqueeze(-1)
|
# row_argmax is FP32-only; the seeded path keeps float64 scores.
|
||||||
|
if scores.dtype == torch.float32:
|
||||||
|
sample_index = row_argmax(scores).unsqueeze(-1)
|
||||||
|
else:
|
||||||
|
sample_index = scores.argmax(dim=-1, keepdim=True)
|
||||||
else:
|
else:
|
||||||
sample_index = scores.topk(num_samples, dim=-1).indices
|
sample_index = scores.topk(num_samples, dim=-1).indices
|
||||||
sample_p = probs.gather(1, sample_index)
|
sample_p = probs.gather(1, sample_index)
|
||||||
@@ -149,6 +218,10 @@ def sample_draft_proposal(
|
|||||||
next_token_logits: torch.Tensor,
|
next_token_logits: torch.Tensor,
|
||||||
temperatures: torch.Tensor,
|
temperatures: torch.Tensor,
|
||||||
top_ks: Optional[torch.Tensor] = None,
|
top_ks: Optional[torch.Tensor] = None,
|
||||||
|
*,
|
||||||
|
sampling_seed: Optional[torch.Tensor] = None,
|
||||||
|
positions: Optional[torch.Tensor] = None,
|
||||||
|
draft_step: int = 0,
|
||||||
):
|
):
|
||||||
"""Leviathan draft proposal: q = softmax(logits / T), X ~ q.
|
"""Leviathan draft proposal: q = softmax(logits / T), X ~ q.
|
||||||
|
|
||||||
@@ -171,7 +244,13 @@ def sample_draft_proposal(
|
|||||||
which is what greedy means. Drop that renorm and this stops holding.
|
which is what greedy means. Drop that renorm and this stops holding.
|
||||||
"""
|
"""
|
||||||
probs = sampling_softmax(next_token_logits, temperatures=temperatures)
|
probs = sampling_softmax(next_token_logits, temperatures=temperatures)
|
||||||
topk_p, topk_index = fast_sample(probs, num_samples=1)
|
topk_p, topk_index = fast_sample(
|
||||||
|
probs,
|
||||||
|
num_samples=1,
|
||||||
|
sampling_seed=sampling_seed,
|
||||||
|
positions=positions,
|
||||||
|
draft_step=draft_step,
|
||||||
|
)
|
||||||
if top_ks is not None:
|
if top_ks is not None:
|
||||||
# Assert rather than skip on a device mismatch: a host-side top_ks would
|
# Assert rather than skip on a device mismatch: a host-side top_ks would
|
||||||
# make this correction silently vanish, and the symptom -- draft accept
|
# make this correction silently vanish, and the symptom -- draft accept
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import unittest
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from sglang.srt.environ import envs
|
||||||
|
from sglang.srt.utils import kill_process_tree
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.test_deterministic import (
|
||||||
|
BenchArgs,
|
||||||
|
)
|
||||||
|
from sglang.test.test_deterministic import test_deterministic as run_deterministic
|
||||||
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_URL_FOR_TEST,
|
||||||
|
CustomTestCase,
|
||||||
|
popen_launch_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=180, stage="base-b", runner_config="2-gpu-large")
|
||||||
|
|
||||||
|
QWEN35_MODEL = "Qwen/Qwen3.5-9B"
|
||||||
|
SERVER_LAUNCH_TIMEOUT = 600
|
||||||
|
|
||||||
|
|
||||||
|
class TestQwen35EagleRSDeterministic(CustomTestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
other_args = [
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--tp",
|
||||||
|
"2",
|
||||||
|
"--attention-backend",
|
||||||
|
"triton",
|
||||||
|
"--enable-deterministic-inference",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"NEXTN",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--speculative-use-rejection-sampling",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.8",
|
||||||
|
"--disable-radix-cache",
|
||||||
|
]
|
||||||
|
with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
|
||||||
|
cls.process = popen_launch_server(
|
||||||
|
QWEN35_MODEL,
|
||||||
|
cls.base_url,
|
||||||
|
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||||
|
other_args=other_args,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
kill_process_tree(cls.process.pid)
|
||||||
|
|
||||||
|
def test_seeded_determinism(self):
|
||||||
|
endpoint = urlparse(self.base_url)
|
||||||
|
results = run_deterministic(
|
||||||
|
BenchArgs(
|
||||||
|
host=endpoint.hostname,
|
||||||
|
port=endpoint.port,
|
||||||
|
temperature=1.0,
|
||||||
|
sampling_seed=42,
|
||||||
|
max_new_tokens=32,
|
||||||
|
test_mode="single",
|
||||||
|
n_trials=6,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(results, [1])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+62
@@ -0,0 +1,62 @@
|
|||||||
|
import unittest
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||||
|
from sglang.test.test_deterministic import (
|
||||||
|
BenchArgs,
|
||||||
|
)
|
||||||
|
from sglang.test.test_deterministic import test_deterministic as run_deterministic
|
||||||
|
from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200")
|
||||||
|
|
||||||
|
STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash"
|
||||||
|
|
||||||
|
|
||||||
|
class TestStep3p5FlashMultiLayerEagleRSDeterministic(DefaultServerBase):
|
||||||
|
model = STEP3P5_FLASH_MODEL_PATH
|
||||||
|
timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
|
||||||
|
other_args = [
|
||||||
|
"--tp",
|
||||||
|
"8",
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
|
"--attention-backend",
|
||||||
|
"fa3",
|
||||||
|
"--enable-multi-layer-eagle",
|
||||||
|
"--speculative-use-rejection-sampling",
|
||||||
|
"--enable-deterministic-inference",
|
||||||
|
"--mem-fraction-static",
|
||||||
|
"0.75",
|
||||||
|
"--chunked-prefill-size",
|
||||||
|
"4096",
|
||||||
|
"--model-loader-extra-config",
|
||||||
|
'{"enable_multithread_load": true, "num_threads": 64}',
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_seeded_determinism(self):
|
||||||
|
endpoint = urlparse(self.base_url)
|
||||||
|
results = run_deterministic(
|
||||||
|
BenchArgs(
|
||||||
|
host=endpoint.hostname,
|
||||||
|
port=endpoint.port,
|
||||||
|
temperature=1.0,
|
||||||
|
sampling_seed=42,
|
||||||
|
max_new_tokens=32,
|
||||||
|
test_mode="single",
|
||||||
|
n_trials=6,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(results, [1])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""Tests for seeded, batch-invariant EAGLE draft proposal sampling."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from sglang.srt.speculative.spec_utils import fast_sample
|
||||||
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
from sglang.test.test_utils import CustomTestCase
|
||||||
|
|
||||||
|
register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||||
|
|
||||||
|
|
||||||
|
def _sample(probs, seeds, positions, *, draft_step=0):
|
||||||
|
return fast_sample(
|
||||||
|
probs,
|
||||||
|
sampling_seed=torch.tensor(seeds, device="cuda", dtype=torch.int64),
|
||||||
|
positions=torch.tensor(positions, device="cuda", dtype=torch.int64),
|
||||||
|
draft_step=draft_step,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeterministicFastSample(CustomTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
torch.manual_seed(1234)
|
||||||
|
weights = torch.rand((8, 4096), device="cuda", dtype=torch.float32)
|
||||||
|
self.probs = weights / weights.sum(dim=-1, keepdim=True)
|
||||||
|
self.seeds = [101, 202, 303, 404, 505, 606, 707, 808]
|
||||||
|
self.positions = [11, 12, 13, 14, 15, 16, 17, 18]
|
||||||
|
|
||||||
|
def test_replay_is_bitwise_reproducible(self):
|
||||||
|
p_a, index_a = _sample(self.probs, self.seeds, self.positions)
|
||||||
|
p_b, index_b = _sample(self.probs, self.seeds, self.positions)
|
||||||
|
|
||||||
|
self.assertTrue(torch.equal(index_a, index_b))
|
||||||
|
self.assertTrue(torch.equal(p_a, p_b))
|
||||||
|
|
||||||
|
def test_cuda_graph_replay_and_input_updates(self):
|
||||||
|
seeds = torch.tensor(self.seeds, device="cuda", dtype=torch.int64)
|
||||||
|
positions = torch.tensor(self.positions, device="cuda", dtype=torch.int64)
|
||||||
|
|
||||||
|
# Warm Triton/PyTorch kernels before capture. Use a nonzero step so
|
||||||
|
# graph replay also covers the step-domain column offset.
|
||||||
|
fast_sample(self.probs, sampling_seed=seeds, positions=positions, draft_step=1)
|
||||||
|
graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(graph):
|
||||||
|
graph_p, graph_index = fast_sample(
|
||||||
|
self.probs,
|
||||||
|
sampling_seed=seeds,
|
||||||
|
positions=positions,
|
||||||
|
draft_step=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
graph.replay()
|
||||||
|
first_p = graph_p.clone()
|
||||||
|
first_index = graph_index.clone()
|
||||||
|
graph.replay()
|
||||||
|
self.assertTrue(torch.equal(first_p, graph_p))
|
||||||
|
self.assertTrue(torch.equal(first_index, graph_index))
|
||||||
|
|
||||||
|
seeds.add_(1)
|
||||||
|
positions.add_(2)
|
||||||
|
graph.replay()
|
||||||
|
updated_p = graph_p.clone()
|
||||||
|
updated_index = graph_index.clone()
|
||||||
|
expected_p, expected_index = fast_sample(
|
||||||
|
self.probs,
|
||||||
|
sampling_seed=seeds,
|
||||||
|
positions=positions,
|
||||||
|
draft_step=1,
|
||||||
|
)
|
||||||
|
self.assertTrue(torch.equal(updated_p, expected_p))
|
||||||
|
self.assertTrue(torch.equal(updated_index, expected_index))
|
||||||
|
|
||||||
|
def test_batch_permutation_and_split_are_invariant(self):
|
||||||
|
p_ref, index_ref = _sample(self.probs, self.seeds, self.positions)
|
||||||
|
|
||||||
|
permutation = torch.tensor([6, 2, 7, 0, 4, 1, 5, 3], device="cuda")
|
||||||
|
inverse = torch.argsort(permutation)
|
||||||
|
p_permuted, index_permuted = _sample(
|
||||||
|
self.probs[permutation],
|
||||||
|
[self.seeds[i] for i in permutation.cpu().tolist()],
|
||||||
|
[self.positions[i] for i in permutation.cpu().tolist()],
|
||||||
|
)
|
||||||
|
self.assertTrue(torch.equal(index_ref, index_permuted[inverse]))
|
||||||
|
self.assertTrue(torch.equal(p_ref, p_permuted[inverse]))
|
||||||
|
|
||||||
|
split_indices = []
|
||||||
|
split_probs = []
|
||||||
|
for start, end in ((0, 3), (3, 7), (7, 8)):
|
||||||
|
sampled_p, sampled_index = _sample(
|
||||||
|
self.probs[start:end],
|
||||||
|
self.seeds[start:end],
|
||||||
|
self.positions[start:end],
|
||||||
|
)
|
||||||
|
split_indices.append(sampled_index)
|
||||||
|
split_probs.append(sampled_p)
|
||||||
|
self.assertTrue(torch.equal(index_ref, torch.cat(split_indices)))
|
||||||
|
self.assertTrue(torch.equal(p_ref, torch.cat(split_probs)))
|
||||||
|
|
||||||
|
def test_seed_position_and_draft_step_select_separate_streams(self):
|
||||||
|
uniform_probs = torch.full((64, 4096), 1 / 4096, device="cuda")
|
||||||
|
seeds = list(range(64))
|
||||||
|
positions = list(range(100, 164))
|
||||||
|
|
||||||
|
_, base = _sample(uniform_probs, seeds, positions)
|
||||||
|
_, changed_seed = _sample(
|
||||||
|
uniform_probs, [seed + 1 for seed in seeds], positions
|
||||||
|
)
|
||||||
|
_, changed_position = _sample(
|
||||||
|
uniform_probs, seeds, [position + 1 for position in positions]
|
||||||
|
)
|
||||||
|
_, changed_step = _sample(uniform_probs, seeds, positions, draft_step=1)
|
||||||
|
|
||||||
|
self.assertFalse(torch.equal(base, changed_seed))
|
||||||
|
self.assertFalse(torch.equal(base, changed_position))
|
||||||
|
self.assertFalse(torch.equal(base, changed_step))
|
||||||
|
# Adjacent request seeds must not alias adjacent draft-step streams.
|
||||||
|
self.assertFalse(torch.equal(changed_seed, changed_step))
|
||||||
|
|
||||||
|
def test_seeded_samples_follow_categorical_distribution(self):
|
||||||
|
num_samples = 100_000
|
||||||
|
expected = torch.tensor([0.1, 0.2, 0.7], device="cuda")
|
||||||
|
probs = expected.expand(num_samples, -1)
|
||||||
|
seeds = torch.arange(num_samples, device="cuda", dtype=torch.int64)
|
||||||
|
positions = torch.full((num_samples,), 37, device="cuda", dtype=torch.int64)
|
||||||
|
|
||||||
|
_, sampled = fast_sample(
|
||||||
|
probs,
|
||||||
|
sampling_seed=seeds,
|
||||||
|
positions=positions,
|
||||||
|
)
|
||||||
|
observed = torch.bincount(sampled.view(-1), minlength=3) / num_samples
|
||||||
|
|
||||||
|
self.assertTrue(
|
||||||
|
torch.allclose(observed, expected, rtol=0, atol=0.01),
|
||||||
|
f"Expected frequencies near {expected.tolist()}, got {observed.tolist()}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_hash_endpoints_produce_finite_ordering(self):
|
||||||
|
probs = torch.tensor(
|
||||||
|
[[0.01, 0.09, 0.90], [0.80, 0.15, 0.05]],
|
||||||
|
device="cuda",
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
|
||||||
|
for hash_value in (0, torch.iinfo(torch.uint32).max):
|
||||||
|
with self.subTest(hash_value=hash_value):
|
||||||
|
|
||||||
|
def _constant_hash(seed, positions, col_indices):
|
||||||
|
return torch.full(
|
||||||
|
(seed.shape[0], col_indices.shape[0]),
|
||||||
|
hash_value,
|
||||||
|
device=seed.device,
|
||||||
|
dtype=torch.uint32,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"sglang.kernels.ops.sampling.murmur_hash.murmur_hash32",
|
||||||
|
side_effect=_constant_hash,
|
||||||
|
):
|
||||||
|
_, sampled = _sample(probs, [1, 2], [3, 4])
|
||||||
|
|
||||||
|
# Equal finite exponential variates leave probability ordering
|
||||||
|
# to decide the winner. Endpoint infinities would violate this.
|
||||||
|
self.assertTrue(
|
||||||
|
torch.equal(
|
||||||
|
sampled,
|
||||||
|
torch.tensor([[2], [0]], device="cuda", dtype=torch.int64),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_requires_matching_seed_position_rows(self):
|
||||||
|
with self.assertRaisesRegex(ValueError, "matching shapes"):
|
||||||
|
fast_sample(
|
||||||
|
self.probs[:2],
|
||||||
|
sampling_seed=torch.tensor([1, 2], device="cuda"),
|
||||||
|
positions=torch.tensor([3], device="cuda"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -104,13 +104,20 @@ class TestBlockVerificationArgs(unittest.TestCase):
|
|||||||
({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"),
|
({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"),
|
||||||
({"speculative_accept_threshold_single": 0.5}, "incompatible"),
|
({"speculative_accept_threshold_single": 0.5}, "incompatible"),
|
||||||
({"speculative_accept_threshold_acc": 0.5}, "incompatible"),
|
({"speculative_accept_threshold_acc": 0.5}, "incompatible"),
|
||||||
({"enable_deterministic_inference": True}, "incompatible"),
|
|
||||||
]
|
]
|
||||||
for overrides, message in cases:
|
for overrides, message in cases:
|
||||||
with self.subTest(overrides=overrides):
|
with self.subTest(overrides=overrides):
|
||||||
with self.assertRaisesRegex(ValueError, message):
|
with self.assertRaisesRegex(ValueError, message):
|
||||||
handle_speculative_decoding(self._args(**overrides))
|
handle_speculative_decoding(self._args(**overrides))
|
||||||
|
|
||||||
|
def test_deterministic_inference_supported_on_cuda(self):
|
||||||
|
# Block verification reuses rejection sampling's seeded draft and verify draws,
|
||||||
|
# so deterministic inference must stay allowed on CUDA.
|
||||||
|
args = self._args(enable_deterministic_inference=True)
|
||||||
|
handle_speculative_decoding(args)
|
||||||
|
self.assertTrue(resolution_result(args, "speculative_use_block_verification"))
|
||||||
|
self.assertTrue(resolution_result(args, "speculative_use_rejection_sampling"))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase):
|
|||||||
self.addCleanup(reset_context)
|
self.addCleanup(reset_context)
|
||||||
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
||||||
|
|
||||||
def _build_runner(self, backend):
|
def _build_runner(self, backend, graph_output=None):
|
||||||
runner = EAGLEDraftCudaGraphRunner.__new__(EAGLEDraftCudaGraphRunner)
|
runner = EAGLEDraftCudaGraphRunner.__new__(EAGLEDraftCudaGraphRunner)
|
||||||
runner.deepep_adapter = SimpleNamespace(replay=lambda: None)
|
runner.deepep_adapter = SimpleNamespace(replay=lambda: None)
|
||||||
runner.buffers = SimpleNamespace(
|
runner.buffers = SimpleNamespace(
|
||||||
@@ -102,7 +102,7 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase):
|
|||||||
def replay_graph_stub(shape_key, forward_batch):
|
def replay_graph_stub(shape_key, forward_batch):
|
||||||
# Observe again during graph replay to catch a premature restore.
|
# Observe again during graph replay to catch a premature restore.
|
||||||
backend.observe("graph_replay", forward_batch)
|
backend.observe("graph_replay", forward_batch)
|
||||||
return None
|
return graph_output
|
||||||
|
|
||||||
runner._replay_graph = replay_graph_stub
|
runner._replay_graph = replay_graph_stub
|
||||||
return runner
|
return runner
|
||||||
@@ -202,6 +202,17 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase):
|
|||||||
self.assertIsNone(observation.seq_lens_sum, msg=observation.phase)
|
self.assertIsNone(observation.seq_lens_sum, msg=observation.phase)
|
||||||
self.assertIsNone(forward_batch.seq_lens_sum)
|
self.assertIsNone(forward_batch.seq_lens_sum)
|
||||||
|
|
||||||
|
def test_execute_leaves_graph_output_ownership_to_caller(self):
|
||||||
|
backend = _RecordingDraftBackend()
|
||||||
|
pooled_probs = torch.ones(CAPTURE_BS, NUM_STEPS, 8)
|
||||||
|
graph_output = (None, None, None, pooled_probs)
|
||||||
|
runner = self._build_runner(backend, graph_output=graph_output)
|
||||||
|
forward_batch = self._build_forward_batch([3, 4, 5, 6], 18)
|
||||||
|
|
||||||
|
output = runner.execute(forward_batch)
|
||||||
|
|
||||||
|
self.assertIs(output[-1], pooled_probs)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Unit tests for deterministic EAGLE draft-proposal stream selection."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from contextlib import nullcontext
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker
|
||||||
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
|
from sglang.test.test_utils import CustomTestCase
|
||||||
|
|
||||||
|
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||||
|
|
||||||
|
|
||||||
|
class TestEagleDraftSamplingPosition(CustomTestCase):
|
||||||
|
def test_draft_extend_uses_post_verify_position(self):
|
||||||
|
worker = EagleDraftWorker.__new__(EagleDraftWorker)
|
||||||
|
worker.speculative_num_draft_tokens = 4
|
||||||
|
worker.cuda_graph_runner_for_draft_extend = None
|
||||||
|
worker.plan_stream = None
|
||||||
|
worker.plan_stream_ctx = nullcontext()
|
||||||
|
worker.seed_dsa_topk_from_draft_extend = False
|
||||||
|
worker.topk = 1
|
||||||
|
worker.device = torch.device("cpu")
|
||||||
|
|
||||||
|
logits_output = SimpleNamespace(
|
||||||
|
next_token_logits=torch.zeros(8, 16), hidden_states=None
|
||||||
|
)
|
||||||
|
worker.draft_runner = SimpleNamespace(
|
||||||
|
canary_manager=None,
|
||||||
|
forward=lambda _: SimpleNamespace(logits_output=logits_output),
|
||||||
|
)
|
||||||
|
sampling_info = SimpleNamespace(
|
||||||
|
temperatures=torch.ones(2, 1),
|
||||||
|
top_ks=torch.ones(2, dtype=torch.int32),
|
||||||
|
sampling_seed=torch.tensor([11, 22], dtype=torch.int64),
|
||||||
|
)
|
||||||
|
batch = SimpleNamespace(
|
||||||
|
seq_lens=torch.tensor([10, 20], dtype=torch.int64),
|
||||||
|
sampling_info=sampling_info,
|
||||||
|
)
|
||||||
|
batch_result = SimpleNamespace(
|
||||||
|
logits_output=SimpleNamespace(hidden_states=None),
|
||||||
|
next_token_ids=torch.tensor([1, 2], dtype=torch.int64),
|
||||||
|
accept_lens=torch.tensor([2, 4], dtype=torch.int64),
|
||||||
|
new_seq_lens=torch.tensor([12, 24], dtype=torch.int64),
|
||||||
|
next_draft_input=SimpleNamespace(),
|
||||||
|
prepared_draft_extend_inputs=None,
|
||||||
|
)
|
||||||
|
forward_batch = SimpleNamespace(sampling_info=sampling_info)
|
||||||
|
sample_result = (
|
||||||
|
torch.zeros(2, 16),
|
||||||
|
torch.ones(2, 1),
|
||||||
|
torch.zeros(2, 1, dtype=torch.int64),
|
||||||
|
)
|
||||||
|
resolved_spec = SimpleNamespace(speculative_use_rejection_sampling=True)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"sglang.srt.speculative.eagle_worker_v2.prepare_for_draft_extend",
|
||||||
|
return_value=forward_batch,
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"sglang.srt.speculative.eagle_worker_v2.sample_draft_proposal",
|
||||||
|
return_value=sample_result,
|
||||||
|
) as sample,
|
||||||
|
patch(
|
||||||
|
"sglang.srt.speculative.eagle_worker_v2.get_spec",
|
||||||
|
return_value=resolved_spec,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
worker._draft_extend_for_decode(batch, batch_result)
|
||||||
|
|
||||||
|
sample.assert_called_once()
|
||||||
|
torch.testing.assert_close(
|
||||||
|
sample.call_args.kwargs["positions"], batch_result.new_seq_lens
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user