diff --git a/python/sglang/srt/speculative/dflash_utils.py b/python/sglang/srt/speculative/dflash_utils.py index 4eabf9a8ea..b50bd12518 100644 --- a/python/sglang/srt/speculative/dflash_utils.py +++ b/python/sglang/srt/speculative/dflash_utils.py @@ -956,10 +956,15 @@ def apply_dflash_simulated_acceptance( simulate_acc_len: float, simulate_acc_method: str, simulate_acc_token_mode: str, + next_token_logits: Optional[torch.Tensor] = None, fixed_token_id: int = 100, ) -> None: - """Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob).""" - block_size = candidates.shape[1] + """Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob). + ``next_token_logits`` gives the target argmax when ``target_predict`` is None.""" + bs, block_size = candidates.shape + if simulate_acc_token_mode == "real-draft-token" and target_predict is None: + # The sampling-verify branch does not materialize the target argmax. + target_predict = torch.argmax(next_token_logits, dim=-1).view(bs, block_size) # sample_simulated_acc_len clamps to [1, block_size]. forced_commit_len = sample_simulated_acc_len( diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 22d8ea809c..872e49a61e 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -96,6 +96,7 @@ from sglang.srt.speculative.spec_utils import ( GrammarTree, assign_req_to_token_pool_func, build_grammar_vocab_mask, + check_simulate_acc_token_mode, draft_pp_context, draft_tp_context, ) @@ -361,7 +362,7 @@ class _DominoDraftSampler: lm_head_org_vocab_start=self.lm_head_org_vocab_start, lm_head_num_org=self.lm_head_num_org, lm_head_num_org_padded=self.lm_head_num_org_padded, - prefer_tp_candidate_pool=bs > 1, + in_cuda_graph=True, ) self.out[: bs * (self.block_size - 1)].copy_(proposals.reshape(-1)) @@ -405,6 +406,7 @@ class DFlashWorkerV2(BaseSpecWorker): # Under dp attention, peer DP ranks run different (idle) paths, so # spec broadcasts must stay within the attn-TP group. self._tp_sync = SpecTpSync(get_dp_tp_group()) + check_simulate_acc_token_mode() # Under dp attention, the draft worker runs on the per-DP attn-TP # group, independent of idle peer DP ranks; it is built and run under @@ -2214,7 +2216,24 @@ class DFlashWorkerV2(BaseSpecWorker): target_predict=target_predict, ) out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus) - return accept_len, commit_lens, bonus, out_tokens, new_seq_lens, target_predict + if SIMULATE_ACC_LEN > 0: + apply_dflash_simulated_acceptance( + candidates=candidates, + target_predict=target_predict, + accept_len=accept_len, + commit_lens=commit_lens, + bonus=bonus, + out_tokens=out_tokens, + simulate_acc_len=SIMULATE_ACC_LEN, + simulate_acc_method=SIMULATE_ACC_METHOD, + simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE, + next_token_logits=next_token_logits, + ) + # The Triton path may have written new_seq_lens from the real + # accept_len; recompute it from the forced commit_lens. + new_seq_lens = None + + return accept_len, commit_lens, bonus, out_tokens, new_seq_lens def _validate_phase1_sampling_support(self, batch: ScheduleBatch) -> None: sampling_info = batch.sampling_info @@ -2655,6 +2674,7 @@ class DFlashWorkerV2(BaseSpecWorker): lm_head_num_org_padded=( int(shard.num_org_elements_padded) if shard is not None else None ), + in_cuda_graph=False, ) elif self._draft_sampler is not None and draft_out.can_run_graph: draft_next = self._draft_sampler.out[ @@ -2834,7 +2854,6 @@ class DFlashWorkerV2(BaseSpecWorker): bonus, out_tokens, new_seq_lens, - target_predict, ) = self._accept_block( candidates=candidates, next_token_logits=logits_output.next_token_logits, @@ -2844,34 +2863,6 @@ class DFlashWorkerV2(BaseSpecWorker): bs=bs, ) - if SIMULATE_ACC_LEN > 0: - if SIMULATE_ACC_TOKEN_MODE not in ("fixed", "real-draft-token"): - raise ValueError( - "Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE " - f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or " - "'real-draft-token'." - ) - - if SIMULATE_ACC_TOKEN_MODE == "real-draft-token" and target_predict is None: - # The sampling-verify branch does not materialize the target argmax. - target_predict = torch.argmax( - logits_output.next_token_logits, dim=-1 - ).view(bs, int(self.block_size)) - apply_dflash_simulated_acceptance( - candidates=candidates, - target_predict=target_predict, - accept_len=accept_len, - commit_lens=commit_lens, - bonus=bonus, - out_tokens=out_tokens, - simulate_acc_len=SIMULATE_ACC_LEN, - simulate_acc_method=SIMULATE_ACC_METHOD, - simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE, - ) - # The Triton path may have written new_seq_lens from the real - # accept_len; recompute it from the forced commit_lens. - new_seq_lens = None - sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits( sampling_info, next_token_logits=logits_output.next_token_logits, diff --git a/python/sglang/srt/speculative/domino_utils.py b/python/sglang/srt/speculative/domino_utils.py index 6525c03c97..d44beeb45d 100644 --- a/python/sglang/srt/speculative/domino_utils.py +++ b/python/sglang/srt/speculative/domino_utils.py @@ -19,11 +19,20 @@ def _domino_gru_cell( ) -# Throughput policy for the logical full-vocabulary tensor. TP>1 uses the -# compact candidate pool when this gathered representation would be large. +# A full-vocab gather is one collective; the candidate pool takes four +# all-gathers and an all-reduce. Eager stays launch bound until the gathered +# logits are large, while a captured graph pays only GPU time. _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES = 96 * 1024 * 1024 +def _domino_prefers_tp_candidate_pool( + *, batch_size: int, full_base_logits_bytes: int, in_cuda_graph: bool +) -> bool: + if in_cuda_graph: + return batch_size > 1 + return full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES + + def _domino_tp_first_ids( local_logits: torch.Tensor, *, @@ -315,7 +324,7 @@ def domino_greedy_rollout( lm_head_org_vocab_start: int = 0, lm_head_num_org: int | None = None, lm_head_num_org_padded: int | None = None, - prefer_tp_candidate_pool: bool | None = None, + in_cuda_graph: bool, ) -> torch.Tensor: """Generate a Domino chain using one block-shared base-logit candidate pool.""" if draft_hidden.ndim != 3: @@ -377,17 +386,17 @@ def domino_greedy_rollout( local_logits = F.linear(logits_input, weight).view( num_proposals, batch_size, local_vocab_size ) - if prefer_tp_candidate_pool is None: - full_base_logits_bytes = ( - num_proposals * batch_size * int(vocab_size) * local_logits.element_size() - ) - prefer_tp_candidate_pool = ( - full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES - ) use_tp_candidate_pool = ( tp_size > 1 - and prefer_tp_candidate_pool and 0 < candidate_pool_size < int(vocab_size) + and _domino_prefers_tp_candidate_pool( + batch_size=batch_size, + full_base_logits_bytes=num_proposals + * batch_size + * int(vocab_size) + * local_logits.element_size(), + in_cuda_graph=in_cuda_graph, + ) ) first_ids = None candidate_ids = None diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index e39826167f..ab2a3ccb4a 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -211,6 +211,18 @@ SIMULATE_ACC_LEN = envs.SGLANG_SIMULATE_ACC_LEN.get() # turn off if < 0 SIMULATE_ACC_METHOD = envs.SGLANG_SIMULATE_ACC_METHOD.get() SIMULATE_ACC_TOKEN_MODE = envs.SGLANG_SIMULATE_ACC_TOKEN_MODE.get() + +def check_simulate_acc_token_mode() -> None: + if SIMULATE_ACC_LEN > 0 and SIMULATE_ACC_TOKEN_MODE not in ( + "fixed", + "real-draft-token", + ): + raise ValueError( + "Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE " + f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or 'real-draft-token'." + ) + + TREE_TRAVERSE_TIME_THRESHOLD = 1 # TODO: set this properly TREE_SPEC_KERNEL_AVAILABLE = ( _is_cuda or _is_musa diff --git a/test/registered/kernels/ops/speculative/test_dflash_domino.py b/test/registered/kernels/ops/speculative/test_dflash_domino.py index 1d145ed48a..8967d178cd 100644 --- a/test/registered/kernels/ops/speculative/test_dflash_domino.py +++ b/test/registered/kernels/ops/speculative/test_dflash_domino.py @@ -39,6 +39,7 @@ class TestDFlashDominoRollout(CustomTestCase): vocab_size=31, shift_label=shift_label, candidate_pool_size=pool_size, + in_cuda_graph=False, ) def test_gru_feedback_matches_sequence(self): diff --git a/test/registered/unit/spec/test_dflash_logits.py b/test/registered/unit/spec/test_dflash_logits.py index d9a2d2a874..2b25e3113b 100644 --- a/test/registered/unit/spec/test_dflash_logits.py +++ b/test/registered/unit/spec/test_dflash_logits.py @@ -340,10 +340,16 @@ def test_selector_accept_uses_greedy_fallback_without_staged_sample(monkeypatch) monkeypatch.setattr( worker_mod, "is_dflash_sampling_verify_available", lambda: False ) + observed = {} + + def fake_correct_drafts_and_bonus(**kwargs): + observed.update(kwargs) + return torch.tensor([0]), torch.tensor([7]) + monkeypatch.setattr( worker_mod, "compute_dflash_correct_drafts_and_bonus", - lambda **kwargs: (torch.tensor([0]), torch.tensor([7])), + fake_correct_drafts_and_bonus, ) sync_sites = [] @@ -367,12 +373,12 @@ def test_selector_accept_uses_greedy_fallback_without_staged_sample(monkeypatch) bs=1, ) - accept_len, commit_lens, bonus, out_tokens, _, target_predict = result + accept_len, commit_lens, bonus, out_tokens, _ = result assert accept_len.tolist() == [0] assert commit_lens.tolist() == [1] assert bonus.tolist() == [7] assert out_tokens.tolist() == [[7, 0]] - assert target_predict.tolist() == [[1, 0]] + assert observed["target_predict"].tolist() == [[1, 0]] assert sync_sites == [worker_mod.SpecTpSyncSite.DFLASH_ACCEPT_GREEDY]