[DFlash] Decide the Domino candidate pool in one place and apply simulated acceptance inside the accept (#43791)

This commit is contained in:
Liangsheng Yin
2026-10-12 11:35:57 -07:00
committed by GitHub
parent 392039d752
commit 0efbb3d2b9
6 changed files with 71 additions and 47 deletions
@@ -956,10 +956,15 @@ def apply_dflash_simulated_acceptance(
simulate_acc_len: float,
simulate_acc_method: str,
simulate_acc_token_mode: str,
next_token_logits: Optional[torch.Tensor] = None,
fixed_token_id: int = 100,
) -> None:
"""Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob)."""
block_size = candidates.shape[1]
"""Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob).
``next_token_logits`` gives the target argmax when ``target_predict`` is None."""
bs, block_size = candidates.shape
if simulate_acc_token_mode == "real-draft-token" and target_predict is None:
# The sampling-verify branch does not materialize the target argmax.
target_predict = torch.argmax(next_token_logits, dim=-1).view(bs, block_size)
# sample_simulated_acc_len clamps to [1, block_size].
forced_commit_len = sample_simulated_acc_len(
@@ -96,6 +96,7 @@ from sglang.srt.speculative.spec_utils import (
GrammarTree,
assign_req_to_token_pool_func,
build_grammar_vocab_mask,
check_simulate_acc_token_mode,
draft_pp_context,
draft_tp_context,
)
@@ -361,7 +362,7 @@ class _DominoDraftSampler:
lm_head_org_vocab_start=self.lm_head_org_vocab_start,
lm_head_num_org=self.lm_head_num_org,
lm_head_num_org_padded=self.lm_head_num_org_padded,
prefer_tp_candidate_pool=bs > 1,
in_cuda_graph=True,
)
self.out[: bs * (self.block_size - 1)].copy_(proposals.reshape(-1))
@@ -405,6 +406,7 @@ class DFlashWorkerV2(BaseSpecWorker):
# Under dp attention, peer DP ranks run different (idle) paths, so
# spec broadcasts must stay within the attn-TP group.
self._tp_sync = SpecTpSync(get_dp_tp_group())
check_simulate_acc_token_mode()
# Under dp attention, the draft worker runs on the per-DP attn-TP
# group, independent of idle peer DP ranks; it is built and run under
@@ -2214,7 +2216,24 @@ class DFlashWorkerV2(BaseSpecWorker):
target_predict=target_predict,
)
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
return accept_len, commit_lens, bonus, out_tokens, new_seq_lens, target_predict
if SIMULATE_ACC_LEN > 0:
apply_dflash_simulated_acceptance(
candidates=candidates,
target_predict=target_predict,
accept_len=accept_len,
commit_lens=commit_lens,
bonus=bonus,
out_tokens=out_tokens,
simulate_acc_len=SIMULATE_ACC_LEN,
simulate_acc_method=SIMULATE_ACC_METHOD,
simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE,
next_token_logits=next_token_logits,
)
# The Triton path may have written new_seq_lens from the real
# accept_len; recompute it from the forced commit_lens.
new_seq_lens = None
return accept_len, commit_lens, bonus, out_tokens, new_seq_lens
def _validate_phase1_sampling_support(self, batch: ScheduleBatch) -> None:
sampling_info = batch.sampling_info
@@ -2655,6 +2674,7 @@ class DFlashWorkerV2(BaseSpecWorker):
lm_head_num_org_padded=(
int(shard.num_org_elements_padded) if shard is not None else None
),
in_cuda_graph=False,
)
elif self._draft_sampler is not None and draft_out.can_run_graph:
draft_next = self._draft_sampler.out[
@@ -2834,7 +2854,6 @@ class DFlashWorkerV2(BaseSpecWorker):
bonus,
out_tokens,
new_seq_lens,
target_predict,
) = self._accept_block(
candidates=candidates,
next_token_logits=logits_output.next_token_logits,
@@ -2844,34 +2863,6 @@ class DFlashWorkerV2(BaseSpecWorker):
bs=bs,
)
if SIMULATE_ACC_LEN > 0:
if SIMULATE_ACC_TOKEN_MODE not in ("fixed", "real-draft-token"):
raise ValueError(
"Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE "
f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or "
"'real-draft-token'."
)
if SIMULATE_ACC_TOKEN_MODE == "real-draft-token" and target_predict is None:
# The sampling-verify branch does not materialize the target argmax.
target_predict = torch.argmax(
logits_output.next_token_logits, dim=-1
).view(bs, int(self.block_size))
apply_dflash_simulated_acceptance(
candidates=candidates,
target_predict=target_predict,
accept_len=accept_len,
commit_lens=commit_lens,
bonus=bonus,
out_tokens=out_tokens,
simulate_acc_len=SIMULATE_ACC_LEN,
simulate_acc_method=SIMULATE_ACC_METHOD,
simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE,
)
# The Triton path may have written new_seq_lens from the real
# accept_len; recompute it from the forced commit_lens.
new_seq_lens = None
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
sampling_info,
next_token_logits=logits_output.next_token_logits,
+20 -11
View File
@@ -19,11 +19,20 @@ def _domino_gru_cell(
)
# Throughput policy for the logical full-vocabulary tensor. TP>1 uses the
# compact candidate pool when this gathered representation would be large.
# A full-vocab gather is one collective; the candidate pool takes four
# all-gathers and an all-reduce. Eager stays launch bound until the gathered
# logits are large, while a captured graph pays only GPU time.
_DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES = 96 * 1024 * 1024
def _domino_prefers_tp_candidate_pool(
*, batch_size: int, full_base_logits_bytes: int, in_cuda_graph: bool
) -> bool:
if in_cuda_graph:
return batch_size > 1
return full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES
def _domino_tp_first_ids(
local_logits: torch.Tensor,
*,
@@ -315,7 +324,7 @@ def domino_greedy_rollout(
lm_head_org_vocab_start: int = 0,
lm_head_num_org: int | None = None,
lm_head_num_org_padded: int | None = None,
prefer_tp_candidate_pool: bool | None = None,
in_cuda_graph: bool,
) -> torch.Tensor:
"""Generate a Domino chain using one block-shared base-logit candidate pool."""
if draft_hidden.ndim != 3:
@@ -377,17 +386,17 @@ def domino_greedy_rollout(
local_logits = F.linear(logits_input, weight).view(
num_proposals, batch_size, local_vocab_size
)
if prefer_tp_candidate_pool is None:
full_base_logits_bytes = (
num_proposals * batch_size * int(vocab_size) * local_logits.element_size()
)
prefer_tp_candidate_pool = (
full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES
)
use_tp_candidate_pool = (
tp_size > 1
and prefer_tp_candidate_pool
and 0 < candidate_pool_size < int(vocab_size)
and _domino_prefers_tp_candidate_pool(
batch_size=batch_size,
full_base_logits_bytes=num_proposals
* batch_size
* int(vocab_size)
* local_logits.element_size(),
in_cuda_graph=in_cuda_graph,
)
)
first_ids = None
candidate_ids = None
@@ -211,6 +211,18 @@ SIMULATE_ACC_LEN = envs.SGLANG_SIMULATE_ACC_LEN.get() # turn off if < 0
SIMULATE_ACC_METHOD = envs.SGLANG_SIMULATE_ACC_METHOD.get()
SIMULATE_ACC_TOKEN_MODE = envs.SGLANG_SIMULATE_ACC_TOKEN_MODE.get()
def check_simulate_acc_token_mode() -> None:
if SIMULATE_ACC_LEN > 0 and SIMULATE_ACC_TOKEN_MODE not in (
"fixed",
"real-draft-token",
):
raise ValueError(
"Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE "
f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or 'real-draft-token'."
)
TREE_TRAVERSE_TIME_THRESHOLD = 1 # TODO: set this properly
TREE_SPEC_KERNEL_AVAILABLE = (
_is_cuda or _is_musa