From 25ad569ff93a43ce5c66cd94edbf75e2eb29b779 Mon Sep 17 00:00:00 2001 From: Ajit Mistry <55892788+ajit283@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:04:32 +0200 Subject: [PATCH] [Feat] speculative: Seed rejection-sampling draft proposals for deterministic inference (#33395) Co-authored-by: Ajit Mistry Co-authored-by: Ajit Mistry Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang --- .../bench_deterministic_fast_sample.py | 89 +++++++++ .../sglang/srt/arg_groups/speculative_hook.py | 13 +- .../eagle_draft_cuda_graph_runner.py | 17 ++ python/sglang/srt/speculative/eagle_utils.py | 8 +- .../sglang/srt/speculative/eagle_worker_v2.py | 12 +- ...er_eagle_draft_extend_cuda_graph_runner.py | 33 +++- .../multi_layer_eagle_worker_v2.py | 12 +- python/sglang/srt/speculative/spec_utils.py | 111 +++++++++-- ...est_eagle_reject_sampling_deterministic.py | 76 ++++++++ ...yer_eagle_reject_sampling_deterministic.py | 62 ++++++ .../test_deterministic_fast_sample.py | 184 ++++++++++++++++++ .../unit/spec/test_block_verification_args.py | 9 +- .../test_eagle_draft_cuda_graph_runner.py | 15 +- .../unit/spec/test_eagle_draft_sampling.py | 83 ++++++++ 14 files changed, 686 insertions(+), 38 deletions(-) create mode 100644 benchmark/kernels/sampling/bench_deterministic_fast_sample.py create mode 100644 test/registered/e2e/speculative/test_eagle_reject_sampling_deterministic.py create mode 100644 test/registered/e2e/speculative/test_multi_layer_eagle_reject_sampling_deterministic.py create mode 100644 test/registered/kernels/ops/speculative/test_deterministic_fast_sample.py create mode 100644 test/registered/unit/spec/test_eagle_draft_sampling.py diff --git a/benchmark/kernels/sampling/bench_deterministic_fast_sample.py b/benchmark/kernels/sampling/bench_deterministic_fast_sample.py new file mode 100644 index 0000000000..7b256abe02 --- /dev/null +++ b/benchmark/kernels/sampling/bench_deterministic_fast_sample.py @@ -0,0 +1,89 @@ +"""Benchmark seeded versus unseeded EAGLE draft Gumbel sampling. + +Example: + python benchmark/kernels/sampling/bench_deterministic_fast_sample.py \ + --batch-sizes 1 8 32 128 --vocab-sizes 32000 128256 163840 +""" + +import argparse + +import torch +import triton + +from sglang.srt.speculative.spec_utils import fast_sample + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--batch-sizes", type=int, nargs="+", default=[1, 8, 32, 128]) + parser.add_argument( + "--vocab-sizes", type=int, nargs="+", default=[32000, 128256, 163840] + ) + parser.add_argument("--warmup-ms", type=int, default=100) + parser.add_argument("--rep-ms", type=int, default=500) + parser.add_argument( + "--modes", nargs="+", choices=("eager", "graph"), default=["graph"] + ) + return parser.parse_args() + + +def capture(fn): + fn() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + fn() + return graph.replay + + +def main(): + args = parse_args() + device = torch.device("cuda") + print(f"device={torch.cuda.get_device_name(device)}") + print("mode\tbatch\tvocab\tunseeded_us\tseeded_us\tslowdown") + + for vocab_size in args.vocab_sizes: + for batch_size in args.batch_sizes: + logits = torch.randn( + (batch_size, vocab_size), device=device, dtype=torch.float32 + ) + probs = torch.softmax(logits, dim=-1) + seeds = torch.arange(batch_size, device=device, dtype=torch.int64) + 12345 + positions = ( + torch.arange(batch_size, device=device, dtype=torch.int64) + 4096 + ) + + def unseeded(): + return fast_sample(probs) + + def seeded(): + return fast_sample( + probs, + sampling_seed=seeds, + positions=positions, + draft_step=1, + ) + + # Compile/JIT and populate allocator caches before timing. + unseeded() + seeded() + torch.cuda.synchronize() + + for mode in args.modes: + unseeded_fn = capture(unseeded) if mode == "graph" else unseeded + seeded_fn = capture(seeded) if mode == "graph" else seeded + unseeded_ms = triton.testing.do_bench( + unseeded_fn, warmup=args.warmup_ms, rep=args.rep_ms + ) + seeded_ms = triton.testing.do_bench( + seeded_fn, warmup=args.warmup_ms, rep=args.rep_ms + ) + print( + f"{mode}\t{batch_size}\t{vocab_size}\t" + f"{unseeded_ms * 1000:.1f}\t{seeded_ms * 1000:.1f}\t" + f"{seeded_ms / unseeded_ms:.2f}x" + ) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 40995d45ea..fb9e7912ce 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -1141,6 +1141,12 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: raise ValueError( "--speculative-use-rejection-sampling requires --speculative-eagle-topk=1." ) + if cfg.enable_deterministic_inference and not cfg.device.startswith("cuda"): + raise ValueError( + "Deterministic rejection sampling is currently supported only on " + "CUDA-compatible devices because draft proposal sampling uses a " + "Triton hash kernel." + ) if ( cfg.speculative_accept_threshold_single != 1.0 or cfg.speculative_accept_threshold_acc != 1.0 @@ -1151,13 +1157,6 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "--speculative-accept-threshold-acc; rejection sampling ignores " "the accept thresholds." ) - if cfg.enable_deterministic_inference: - raise ValueError( - "--speculative-use-rejection-sampling is incompatible with " - "--enable-deterministic-inference; the sampling kernel draws " - "coins from the global RNG and is not batch-invariant." - ) - if ( resolved_view(server_args).enable_multi_layer_eagle and cfg.speculative_eagle_topk != 1 diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index f7205cae82..8d57fdd450 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -77,6 +77,7 @@ class EagleDraftInputBuffers(ForwardInputBuffers): topk_p: torch.Tensor topk_index: torch.Tensor draft_probs: Optional[torch.Tensor] + sampling_seed: Optional[torch.Tensor] hidden_states: Optional[torch.Tensor] global_num_tokens_gpu: Optional[torch.Tensor] global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] @@ -203,6 +204,12 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): if get_spec().speculative_use_rejection_sampling else None ) + sampling_seed = ( + torch.zeros((self.max_bs,), dtype=torch.int64) + if get_exec().deterministic.enable_deterministic_inference + and get_spec().speculative_use_rejection_sampling + else None + ) _hidden_size, _hidden_dtype = get_draft_recurrent_hidden_state_spec( model_runner ) @@ -271,6 +278,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): topk_p=topk_p, topk_index=topk_index, draft_probs=draft_probs, + sampling_seed=sampling_seed, hidden_states=hidden_states, global_num_tokens_gpu=global_num_tokens_gpu, global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu, @@ -456,6 +464,11 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): need_top_k_sampling=False, need_min_p_sampling=False, vocab_size=self.model_runner.model_config.vocab_size, + sampling_seed=( + self.buffers.sampling_seed[:num_seqs] + if self.buffers.sampling_seed is not None + else None + ), ) forward_batch = ForwardBatch( @@ -671,6 +684,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): self.temperatures[:raw_bs].copy_( forward_batch.sampling_info.temperatures[:raw_bs] ) + if self.buffers.sampling_seed is not None: + self.buffers.sampling_seed[:raw_bs].copy_( + forward_batch.sampling_info.sampling_seed[:raw_bs] + ) self.top_ks[:raw_bs].copy_(forward_batch.sampling_info.top_ks[:raw_bs]) # TODO(ch-wan): support num_token_non_padded diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 1c1350d40e..631e2d7887 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -428,11 +428,9 @@ def _seeded_verify_coins( per-draft rejection coins; column draft_token_num drives the final fallback-sampling coin. - Scope: this seeds only the verify-side RNG. With rejection sampling the - draft workers still pick candidates via unseeded multinomial - (fast_sample in eagle_worker_v2), so that mode stays non-deterministic - until the draft RNG is seeded in a follow-up; top-k/greedy draft - selection is already deterministic. + Draft proposal draws use a separate domain-separated seeded stream in + ``fast_sample``. Keeping the streams separate preserves the independence + required by rejection sampling. """ from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32 diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 36c1410bf7..b61ceefea5 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -817,6 +817,9 @@ class EagleDraftWorker(EagleDraftWorkerBase): logits_output.next_token_logits, forward_batch.sampling_info.temperatures, forward_batch.sampling_info.top_ks, + sampling_seed=forward_batch.sampling_info.sampling_seed, + positions=forward_batch.positions + 1, + draft_step=i + 1, ) draft_probs_list.append(probs) forward_batch.positions.add_(1) @@ -1031,7 +1034,12 @@ class EagleDraftWorker(EagleDraftWorkerBase): use_rejection_sampling, ) if use_rejection_sampling: - topk_p, topk_index = fast_sample(probs, num_samples=1) + topk_p, topk_index = fast_sample( + probs, + num_samples=1, + sampling_seed=batch.sampling_info.sampling_seed, + positions=batch.seq_lens, + ) else: topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) return EagleDraftInput( @@ -1199,6 +1207,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): draft_logits_output.next_token_logits, batch.sampling_info.temperatures, batch.sampling_info.top_ks, + sampling_seed=batch.sampling_info.sampling_seed, + positions=batch_result.new_seq_lens, ) elif self.topk == 1 and _is_hip: ret_topk_p, ret_topk_index = draft_topk1_argmax_only( diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index bf3ab0a44e..6776269b4b 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -136,6 +136,7 @@ class MultiLayerEagleDraftExtendInputBuffers(ForwardInputBuffers): # of draft_probs selects the in-graph proposal branch in _run_step_body). temperatures: Optional[torch.Tensor] draft_probs: Optional[torch.Tensor] + sampling_seed: Optional[torch.Tensor] class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): @@ -394,7 +395,17 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): buffers.temperatures[:bs], buffers.draft_probs[:bs, self.step], ) - ret.topk_p, ret.topk_index = fast_sample(probs, num_samples=1) + ret.topk_p, ret.topk_index = fast_sample( + probs, + num_samples=1, + sampling_seed=( + buffers.sampling_seed[:bs] + if buffers.sampling_seed is not None + else None + ), + positions=buffers.seq_lens[:bs], + draft_step=self.step, + ) def _run_step_body(self, forward_batch: ForwardBatch, num_tokens: int, bs: int): """One draft step's body: model forward + chain-hidden write + top-k. @@ -693,9 +704,15 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: (max_bs, self.speculative_num_steps, vocab_size), dtype=torch.float, ) + sampling_seed = ( + torch.zeros((max_bs,), dtype=torch.int64) + if get_exec().deterministic.enable_deterministic_inference + else None + ) else: temperatures = None draft_probs = None + sampling_seed = None if self.require_gathered_buffer: if self.require_mlp_tp_gather: @@ -739,6 +756,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu, temperatures=temperatures, draft_probs=draft_probs, + sampling_seed=sampling_seed, ) def _prepare_extra(self, forward_batch: ForwardBatch) -> None: @@ -867,6 +885,11 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: self.seq_len_fill_value, ) + if buffers.sampling_seed is not None: + buffers.sampling_seed[:raw_bs].copy_( + forward_batch.sampling_info.sampling_seed[:raw_bs] + ) + # Refresh the host mirror only when published; hand replay None # otherwise so no consumer reads a stale buffer. if forward_batch.seq_lens_cpu is not None: @@ -968,10 +991,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner( Rejection sampling is supported by sampling X ~ q inside the graph (_sample_draft_proposal, selected by the draft_probs buffer's presence): the captured rotation then carries the sampled token, prepare() stages the - temperatures sampling_info cannot deliver in-graph, and the worker clones - the per-step q off buffers.draft_probs after replay. torch.multinomial - draws through the graph-registered Philox generator, so each replay gets - fresh coins. + temperatures and sampling seeds cannot be delivered in-graph, and the + worker clones the per-step q off buffers.draft_probs after replay. Seeded + draws are keyed by request position and draft step; unseeded draws keep + using the graph-registered Philox generator. """ rotates_in_graph = True diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index ccbf75837b..c6c1547e37 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -705,6 +705,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): probs, topk_p, topk_index = sample_draft_proposal( output.logits_output.next_token_logits, forward_batch.sampling_info.temperatures, + sampling_seed=forward_batch.sampling_info.sampling_seed, + positions=forward_batch.seq_lens, + draft_step=step, ) draft_probs_list.append(probs) else: @@ -912,6 +915,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): probs, ret_topk_p, ret_topk_index = sample_draft_proposal( step_logits, forward_batch.sampling_info.temperatures, + sampling_seed=forward_batch.sampling_info.sampling_seed, + positions=forward_batch.seq_lens, + draft_step=step, ) ret_draft_probs_list.append(probs) if rotates_in_graph: @@ -985,7 +991,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): ] if self.use_rejection_sampling and self.topk == 1: probs, ret_topk_p, ret_topk_index = sample_draft_proposal( - logits_sel, forward_batch.sampling_info.temperatures + logits_sel, + forward_batch.sampling_info.temperatures, + sampling_seed=forward_batch.sampling_info.sampling_seed, + positions=forward_batch.seq_lens, + draft_step=step, ) ret_draft_probs_list.append(probs) else: diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 98459186b5..3b4685664e 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -105,24 +105,93 @@ def resolve_num_tokens_per_req( raise ValueError(f"Unknown speculative phase: {phase}") -def fast_sample(probs: torch.Tensor, num_samples: int = 1): - """Draw from `probs` via the Gumbel-max trick: argmax(probs / Exp(1)). +# Domain-separate draft proposal draws from target sampling and verify coins. +# The per-step offset also separates successive draft draws if their absolute +# position inputs overlap. +_DRAFT_GUMBEL_SEED_SALT = 0x44524146545F0000 - Distributionally equivalent to torch.multinomial, but avoids multinomial's - device-side distribution-validity assert, which the draft CUDA graph would - otherwise capture and replay every step. q is clamped off zero so a zero - draw can't yield inf/NaN scores that argmax would wrongly select; fp32 - avoids bf16 argmax ties biasing the draw. Set SGLANG_OPT_USE_GUMBEL_SAMPLE=0 - to fall back to torch.multinomial. + +def fast_sample( + probs: torch.Tensor, + num_samples: int = 1, + *, + sampling_seed: Optional[torch.Tensor] = None, + positions: Optional[torch.Tensor] = None, + draft_step: int = 0, +): + """Draw from ``probs`` with the exponential-race form of Gumbel-max. + + When ``sampling_seed`` is provided, derive one exponential variate per + request, position, draft step, and vocabulary entry with ``murmur_hash32``. + This makes draft proposals reproducible under batch reordering and CUDA + graph replay. The midpoint uint32 mapping stays strictly inside ``(0, 1)``, + avoiding infinite exponential/Gumbel values at either hash endpoint. + + The unseeded path keeps using PyTorch's fast RNG-backed implementation. + Set ``SGLANG_OPT_USE_GUMBEL_SAMPLE=0`` to make that path fall back to + ``torch.multinomial``. """ - if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get(): - sample_index = torch.multinomial(probs, num_samples=num_samples) - return probs.gather(1, sample_index), sample_index - q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0) - q.clamp_min_(torch.finfo(torch.float32).tiny) - scores = probs.float() / q + if sampling_seed is not None: + if positions is None: + raise ValueError("positions are required for deterministic fast_sample") + if sampling_seed.shape != positions.shape: + raise ValueError( + "sampling_seed and positions must have matching shapes, got " + f"{sampling_seed.shape} and {positions.shape}" + ) + if sampling_seed.numel() != probs.shape[0]: + raise ValueError( + "sampling_seed must have one value per probability row, got " + f"{sampling_seed.numel()} seeds for {probs.shape[0]} rows" + ) + + from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32 + + if draft_step < 0: + raise ValueError(f"draft_step must be non-negative, got {draft_step}") + vocab_size = probs.shape[1] + col_offset = draft_step * vocab_size + if col_offset + vocab_size > 2**32: + raise ValueError( + "draft_step and vocabulary size exceed the uint32 hash-key range: " + f"draft_step={draft_step}, vocab_size={vocab_size}" + ) + + # Encode (draft_step, token_id) in the uint32 column block instead of + # altering the request seed; step zero retains its established stream. + salted_seed = sampling_seed + _DRAFT_GUMBEL_SEED_SALT + col_indices = torch.arange( + col_offset, + col_offset + vocab_size, + dtype=torch.int64, + device=probs.device, + ) + hashed = murmur_hash32( + salted_seed.to(torch.uint64), positions.to(torch.uint64), col_indices + ) + + # If U ~ Uniform(0, 1), then -log(U) ~ Exp(1), and + # argmax_i probs[i] / Exp_i samples from the categorical distribution. + # Keep float64 for this first implementation, matching the established + # deterministic sampler's numerical-conservatism policy. + scores = hashed.to(torch.float64) + del hashed + scores.add_(0.5).div_(2.0**32).log_().neg_().reciprocal_() + scores.mul_(probs) + else: + if not envs.SGLANG_OPT_USE_GUMBEL_SAMPLE.get(): + sample_index = torch.multinomial(probs, num_samples=num_samples) + return probs.gather(1, sample_index), sample_index + q = torch.empty_like(probs, dtype=torch.float32).exponential_(1.0) + q.clamp_min_(torch.finfo(torch.float32).tiny) + scores = probs.float() / q + if num_samples == 1: - sample_index = row_argmax(scores).unsqueeze(-1) + # row_argmax is FP32-only; the seeded path keeps float64 scores. + if scores.dtype == torch.float32: + sample_index = row_argmax(scores).unsqueeze(-1) + else: + sample_index = scores.argmax(dim=-1, keepdim=True) else: sample_index = scores.topk(num_samples, dim=-1).indices sample_p = probs.gather(1, sample_index) @@ -149,6 +218,10 @@ def sample_draft_proposal( next_token_logits: torch.Tensor, temperatures: torch.Tensor, top_ks: Optional[torch.Tensor] = None, + *, + sampling_seed: Optional[torch.Tensor] = None, + positions: Optional[torch.Tensor] = None, + draft_step: int = 0, ): """Leviathan draft proposal: q = softmax(logits / T), X ~ q. @@ -171,7 +244,13 @@ def sample_draft_proposal( which is what greedy means. Drop that renorm and this stops holding. """ probs = sampling_softmax(next_token_logits, temperatures=temperatures) - topk_p, topk_index = fast_sample(probs, num_samples=1) + topk_p, topk_index = fast_sample( + probs, + num_samples=1, + sampling_seed=sampling_seed, + positions=positions, + draft_step=draft_step, + ) if top_ks is not None: # Assert rather than skip on a device mismatch: a host-side top_ks would # make this correction silently vanish, and the symptom -- draft accept diff --git a/test/registered/e2e/speculative/test_eagle_reject_sampling_deterministic.py b/test/registered/e2e/speculative/test_eagle_reject_sampling_deterministic.py new file mode 100644 index 0000000000..1bb060c979 --- /dev/null +++ b/test/registered/e2e/speculative/test_eagle_reject_sampling_deterministic.py @@ -0,0 +1,76 @@ +import unittest +from urllib.parse import urlparse + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_deterministic import ( + BenchArgs, +) +from sglang.test.test_deterministic import test_deterministic as run_deterministic +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=180, stage="base-b", runner_config="2-gpu-large") + +QWEN35_MODEL = "Qwen/Qwen3.5-9B" +SERVER_LAUNCH_TIMEOUT = 600 + + +class TestQwen35EagleRSDeterministic(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "2", + "--attention-backend", + "triton", + "--enable-deterministic-inference", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--speculative-use-rejection-sampling", + "--mem-fraction-static", + "0.8", + "--disable-radix-cache", + ] + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): + cls.process = popen_launch_server( + QWEN35_MODEL, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_seeded_determinism(self): + endpoint = urlparse(self.base_url) + results = run_deterministic( + BenchArgs( + host=endpoint.hostname, + port=endpoint.port, + temperature=1.0, + sampling_seed=42, + max_new_tokens=32, + test_mode="single", + n_trials=6, + ) + ) + self.assertEqual(results, [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/e2e/speculative/test_multi_layer_eagle_reject_sampling_deterministic.py b/test/registered/e2e/speculative/test_multi_layer_eagle_reject_sampling_deterministic.py new file mode 100644 index 0000000000..9b896ba901 --- /dev/null +++ b/test/registered/e2e/speculative/test_multi_layer_eagle_reject_sampling_deterministic.py @@ -0,0 +1,62 @@ +import unittest +from urllib.parse import urlparse + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_deterministic import ( + BenchArgs, +) +from sglang.test.test_deterministic import test_deterministic as run_deterministic +from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + +register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200") + +STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash" + + +class TestStep3p5FlashMultiLayerEagleRSDeterministic(DefaultServerBase): + model = STEP3P5_FLASH_MODEL_PATH + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3 + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--attention-backend", + "fa3", + "--enable-multi-layer-eagle", + "--speculative-use-rejection-sampling", + "--enable-deterministic-inference", + "--mem-fraction-static", + "0.75", + "--chunked-prefill-size", + "4096", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + + def test_seeded_determinism(self): + endpoint = urlparse(self.base_url) + results = run_deterministic( + BenchArgs( + host=endpoint.hostname, + port=endpoint.port, + temperature=1.0, + sampling_seed=42, + max_new_tokens=32, + test_mode="single", + n_trials=6, + ) + ) + self.assertEqual(results, [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_deterministic_fast_sample.py b/test/registered/kernels/ops/speculative/test_deterministic_fast_sample.py new file mode 100644 index 0000000000..21e15467e1 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_deterministic_fast_sample.py @@ -0,0 +1,184 @@ +"""Tests for seeded, batch-invariant EAGLE draft proposal sampling.""" + +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.speculative.spec_utils import fast_sample +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def _sample(probs, seeds, positions, *, draft_step=0): + return fast_sample( + probs, + sampling_seed=torch.tensor(seeds, device="cuda", dtype=torch.int64), + positions=torch.tensor(positions, device="cuda", dtype=torch.int64), + draft_step=draft_step, + ) + + +class TestDeterministicFastSample(CustomTestCase): + def setUp(self): + torch.manual_seed(1234) + weights = torch.rand((8, 4096), device="cuda", dtype=torch.float32) + self.probs = weights / weights.sum(dim=-1, keepdim=True) + self.seeds = [101, 202, 303, 404, 505, 606, 707, 808] + self.positions = [11, 12, 13, 14, 15, 16, 17, 18] + + def test_replay_is_bitwise_reproducible(self): + p_a, index_a = _sample(self.probs, self.seeds, self.positions) + p_b, index_b = _sample(self.probs, self.seeds, self.positions) + + self.assertTrue(torch.equal(index_a, index_b)) + self.assertTrue(torch.equal(p_a, p_b)) + + def test_cuda_graph_replay_and_input_updates(self): + seeds = torch.tensor(self.seeds, device="cuda", dtype=torch.int64) + positions = torch.tensor(self.positions, device="cuda", dtype=torch.int64) + + # Warm Triton/PyTorch kernels before capture. Use a nonzero step so + # graph replay also covers the step-domain column offset. + fast_sample(self.probs, sampling_seed=seeds, positions=positions, draft_step=1) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_p, graph_index = fast_sample( + self.probs, + sampling_seed=seeds, + positions=positions, + draft_step=1, + ) + + graph.replay() + first_p = graph_p.clone() + first_index = graph_index.clone() + graph.replay() + self.assertTrue(torch.equal(first_p, graph_p)) + self.assertTrue(torch.equal(first_index, graph_index)) + + seeds.add_(1) + positions.add_(2) + graph.replay() + updated_p = graph_p.clone() + updated_index = graph_index.clone() + expected_p, expected_index = fast_sample( + self.probs, + sampling_seed=seeds, + positions=positions, + draft_step=1, + ) + self.assertTrue(torch.equal(updated_p, expected_p)) + self.assertTrue(torch.equal(updated_index, expected_index)) + + def test_batch_permutation_and_split_are_invariant(self): + p_ref, index_ref = _sample(self.probs, self.seeds, self.positions) + + permutation = torch.tensor([6, 2, 7, 0, 4, 1, 5, 3], device="cuda") + inverse = torch.argsort(permutation) + p_permuted, index_permuted = _sample( + self.probs[permutation], + [self.seeds[i] for i in permutation.cpu().tolist()], + [self.positions[i] for i in permutation.cpu().tolist()], + ) + self.assertTrue(torch.equal(index_ref, index_permuted[inverse])) + self.assertTrue(torch.equal(p_ref, p_permuted[inverse])) + + split_indices = [] + split_probs = [] + for start, end in ((0, 3), (3, 7), (7, 8)): + sampled_p, sampled_index = _sample( + self.probs[start:end], + self.seeds[start:end], + self.positions[start:end], + ) + split_indices.append(sampled_index) + split_probs.append(sampled_p) + self.assertTrue(torch.equal(index_ref, torch.cat(split_indices))) + self.assertTrue(torch.equal(p_ref, torch.cat(split_probs))) + + def test_seed_position_and_draft_step_select_separate_streams(self): + uniform_probs = torch.full((64, 4096), 1 / 4096, device="cuda") + seeds = list(range(64)) + positions = list(range(100, 164)) + + _, base = _sample(uniform_probs, seeds, positions) + _, changed_seed = _sample( + uniform_probs, [seed + 1 for seed in seeds], positions + ) + _, changed_position = _sample( + uniform_probs, seeds, [position + 1 for position in positions] + ) + _, changed_step = _sample(uniform_probs, seeds, positions, draft_step=1) + + self.assertFalse(torch.equal(base, changed_seed)) + self.assertFalse(torch.equal(base, changed_position)) + self.assertFalse(torch.equal(base, changed_step)) + # Adjacent request seeds must not alias adjacent draft-step streams. + self.assertFalse(torch.equal(changed_seed, changed_step)) + + def test_seeded_samples_follow_categorical_distribution(self): + num_samples = 100_000 + expected = torch.tensor([0.1, 0.2, 0.7], device="cuda") + probs = expected.expand(num_samples, -1) + seeds = torch.arange(num_samples, device="cuda", dtype=torch.int64) + positions = torch.full((num_samples,), 37, device="cuda", dtype=torch.int64) + + _, sampled = fast_sample( + probs, + sampling_seed=seeds, + positions=positions, + ) + observed = torch.bincount(sampled.view(-1), minlength=3) / num_samples + + self.assertTrue( + torch.allclose(observed, expected, rtol=0, atol=0.01), + f"Expected frequencies near {expected.tolist()}, got {observed.tolist()}", + ) + + def test_hash_endpoints_produce_finite_ordering(self): + probs = torch.tensor( + [[0.01, 0.09, 0.90], [0.80, 0.15, 0.05]], + device="cuda", + dtype=torch.float32, + ) + + for hash_value in (0, torch.iinfo(torch.uint32).max): + with self.subTest(hash_value=hash_value): + + def _constant_hash(seed, positions, col_indices): + return torch.full( + (seed.shape[0], col_indices.shape[0]), + hash_value, + device=seed.device, + dtype=torch.uint32, + ) + + with patch( + "sglang.kernels.ops.sampling.murmur_hash.murmur_hash32", + side_effect=_constant_hash, + ): + _, sampled = _sample(probs, [1, 2], [3, 4]) + + # Equal finite exponential variates leave probability ordering + # to decide the winner. Endpoint infinities would violate this. + self.assertTrue( + torch.equal( + sampled, + torch.tensor([[2], [0]], device="cuda", dtype=torch.int64), + ) + ) + + def test_requires_matching_seed_position_rows(self): + with self.assertRaisesRegex(ValueError, "matching shapes"): + fast_sample( + self.probs[:2], + sampling_seed=torch.tensor([1, 2], device="cuda"), + positions=torch.tensor([3], device="cuda"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_block_verification_args.py b/test/registered/unit/spec/test_block_verification_args.py index c3dc876627..2f246ced2a 100644 --- a/test/registered/unit/spec/test_block_verification_args.py +++ b/test/registered/unit/spec/test_block_verification_args.py @@ -104,13 +104,20 @@ class TestBlockVerificationArgs(unittest.TestCase): ({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"), ({"speculative_accept_threshold_single": 0.5}, "incompatible"), ({"speculative_accept_threshold_acc": 0.5}, "incompatible"), - ({"enable_deterministic_inference": True}, "incompatible"), ] for overrides, message in cases: with self.subTest(overrides=overrides): with self.assertRaisesRegex(ValueError, message): handle_speculative_decoding(self._args(**overrides)) + def test_deterministic_inference_supported_on_cuda(self): + # Block verification reuses rejection sampling's seeded draft and verify draws, + # so deterministic inference must stay allowed on CUDA. + args = self._args(enable_deterministic_inference=True) + handle_speculative_decoding(args) + self.assertTrue(resolution_result(args, "speculative_use_block_verification")) + self.assertTrue(resolution_result(args, "speculative_use_rejection_sampling")) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/spec/test_eagle_draft_cuda_graph_runner.py b/test/registered/unit/spec/test_eagle_draft_cuda_graph_runner.py index 25dfe2adcc..388c5d09bd 100644 --- a/test/registered/unit/spec/test_eagle_draft_cuda_graph_runner.py +++ b/test/registered/unit/spec/test_eagle_draft_cuda_graph_runner.py @@ -67,7 +67,7 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase): self.addCleanup(reset_context) publish(ServerArgs(model_path="dummy"), role="tokenizer") - def _build_runner(self, backend): + def _build_runner(self, backend, graph_output=None): runner = EAGLEDraftCudaGraphRunner.__new__(EAGLEDraftCudaGraphRunner) runner.deepep_adapter = SimpleNamespace(replay=lambda: None) runner.buffers = SimpleNamespace( @@ -102,7 +102,7 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase): def replay_graph_stub(shape_key, forward_batch): # Observe again during graph replay to catch a premature restore. backend.observe("graph_replay", forward_batch) - return None + return graph_output runner._replay_graph = replay_graph_stub return runner @@ -202,6 +202,17 @@ class TestEagleDraftCudaGraphRunner(CustomTestCase): self.assertIsNone(observation.seq_lens_sum, msg=observation.phase) self.assertIsNone(forward_batch.seq_lens_sum) + def test_execute_leaves_graph_output_ownership_to_caller(self): + backend = _RecordingDraftBackend() + pooled_probs = torch.ones(CAPTURE_BS, NUM_STEPS, 8) + graph_output = (None, None, None, pooled_probs) + runner = self._build_runner(backend, graph_output=graph_output) + forward_batch = self._build_forward_batch([3, 4, 5, 6], 18) + + output = runner.execute(forward_batch) + + self.assertIs(output[-1], pooled_probs) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/spec/test_eagle_draft_sampling.py b/test/registered/unit/spec/test_eagle_draft_sampling.py new file mode 100644 index 0000000000..8ea2ca6f1e --- /dev/null +++ b/test/registered/unit/spec/test_eagle_draft_sampling.py @@ -0,0 +1,83 @@ +"""Unit tests for deterministic EAGLE draft-proposal stream selection.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestEagleDraftSamplingPosition(CustomTestCase): + def test_draft_extend_uses_post_verify_position(self): + worker = EagleDraftWorker.__new__(EagleDraftWorker) + worker.speculative_num_draft_tokens = 4 + worker.cuda_graph_runner_for_draft_extend = None + worker.plan_stream = None + worker.plan_stream_ctx = nullcontext() + worker.seed_dsa_topk_from_draft_extend = False + worker.topk = 1 + worker.device = torch.device("cpu") + + logits_output = SimpleNamespace( + next_token_logits=torch.zeros(8, 16), hidden_states=None + ) + worker.draft_runner = SimpleNamespace( + canary_manager=None, + forward=lambda _: SimpleNamespace(logits_output=logits_output), + ) + sampling_info = SimpleNamespace( + temperatures=torch.ones(2, 1), + top_ks=torch.ones(2, dtype=torch.int32), + sampling_seed=torch.tensor([11, 22], dtype=torch.int64), + ) + batch = SimpleNamespace( + seq_lens=torch.tensor([10, 20], dtype=torch.int64), + sampling_info=sampling_info, + ) + batch_result = SimpleNamespace( + logits_output=SimpleNamespace(hidden_states=None), + next_token_ids=torch.tensor([1, 2], dtype=torch.int64), + accept_lens=torch.tensor([2, 4], dtype=torch.int64), + new_seq_lens=torch.tensor([12, 24], dtype=torch.int64), + next_draft_input=SimpleNamespace(), + prepared_draft_extend_inputs=None, + ) + forward_batch = SimpleNamespace(sampling_info=sampling_info) + sample_result = ( + torch.zeros(2, 16), + torch.ones(2, 1), + torch.zeros(2, 1, dtype=torch.int64), + ) + resolved_spec = SimpleNamespace(speculative_use_rejection_sampling=True) + + with ( + patch( + "sglang.srt.speculative.eagle_worker_v2.prepare_for_draft_extend", + return_value=forward_batch, + ), + patch( + "sglang.srt.speculative.eagle_worker_v2.sample_draft_proposal", + return_value=sample_result, + ) as sample, + patch( + "sglang.srt.speculative.eagle_worker_v2.get_spec", + return_value=resolved_spec, + ), + ): + worker._draft_extend_for_decode(batch, batch_result) + + sample.assert_called_once() + torch.testing.assert_close( + sample.call_args.kwargs["positions"], batch_result.new_seq_lens + ) + + +if __name__ == "__main__": + unittest.main()