[DFlash] Decide the Domino candidate pool in one place and apply simulated acceptance inside the accept (#43791)
This commit is contained in:
@@ -956,10 +956,15 @@ def apply_dflash_simulated_acceptance(
|
||||
simulate_acc_len: float,
|
||||
simulate_acc_method: str,
|
||||
simulate_acc_token_mode: str,
|
||||
next_token_logits: Optional[torch.Tensor] = None,
|
||||
fixed_token_id: int = 100,
|
||||
) -> None:
|
||||
"""Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob)."""
|
||||
block_size = candidates.shape[1]
|
||||
"""Forces the DFlash acceptance length (SGLANG_SIMULATE_ACC_LEN benchmark knob).
|
||||
``next_token_logits`` gives the target argmax when ``target_predict`` is None."""
|
||||
bs, block_size = candidates.shape
|
||||
if simulate_acc_token_mode == "real-draft-token" and target_predict is None:
|
||||
# The sampling-verify branch does not materialize the target argmax.
|
||||
target_predict = torch.argmax(next_token_logits, dim=-1).view(bs, block_size)
|
||||
|
||||
# sample_simulated_acc_len clamps to [1, block_size].
|
||||
forced_commit_len = sample_simulated_acc_len(
|
||||
|
||||
@@ -96,6 +96,7 @@ from sglang.srt.speculative.spec_utils import (
|
||||
GrammarTree,
|
||||
assign_req_to_token_pool_func,
|
||||
build_grammar_vocab_mask,
|
||||
check_simulate_acc_token_mode,
|
||||
draft_pp_context,
|
||||
draft_tp_context,
|
||||
)
|
||||
@@ -361,7 +362,7 @@ class _DominoDraftSampler:
|
||||
lm_head_org_vocab_start=self.lm_head_org_vocab_start,
|
||||
lm_head_num_org=self.lm_head_num_org,
|
||||
lm_head_num_org_padded=self.lm_head_num_org_padded,
|
||||
prefer_tp_candidate_pool=bs > 1,
|
||||
in_cuda_graph=True,
|
||||
)
|
||||
self.out[: bs * (self.block_size - 1)].copy_(proposals.reshape(-1))
|
||||
|
||||
@@ -405,6 +406,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# Under dp attention, peer DP ranks run different (idle) paths, so
|
||||
# spec broadcasts must stay within the attn-TP group.
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
check_simulate_acc_token_mode()
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
@@ -2214,7 +2216,24 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
target_predict=target_predict,
|
||||
)
|
||||
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
|
||||
return accept_len, commit_lens, bonus, out_tokens, new_seq_lens, target_predict
|
||||
if SIMULATE_ACC_LEN > 0:
|
||||
apply_dflash_simulated_acceptance(
|
||||
candidates=candidates,
|
||||
target_predict=target_predict,
|
||||
accept_len=accept_len,
|
||||
commit_lens=commit_lens,
|
||||
bonus=bonus,
|
||||
out_tokens=out_tokens,
|
||||
simulate_acc_len=SIMULATE_ACC_LEN,
|
||||
simulate_acc_method=SIMULATE_ACC_METHOD,
|
||||
simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE,
|
||||
next_token_logits=next_token_logits,
|
||||
)
|
||||
# The Triton path may have written new_seq_lens from the real
|
||||
# accept_len; recompute it from the forced commit_lens.
|
||||
new_seq_lens = None
|
||||
|
||||
return accept_len, commit_lens, bonus, out_tokens, new_seq_lens
|
||||
|
||||
def _validate_phase1_sampling_support(self, batch: ScheduleBatch) -> None:
|
||||
sampling_info = batch.sampling_info
|
||||
@@ -2655,6 +2674,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
lm_head_num_org_padded=(
|
||||
int(shard.num_org_elements_padded) if shard is not None else None
|
||||
),
|
||||
in_cuda_graph=False,
|
||||
)
|
||||
elif self._draft_sampler is not None and draft_out.can_run_graph:
|
||||
draft_next = self._draft_sampler.out[
|
||||
@@ -2834,7 +2854,6 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
bonus,
|
||||
out_tokens,
|
||||
new_seq_lens,
|
||||
target_predict,
|
||||
) = self._accept_block(
|
||||
candidates=candidates,
|
||||
next_token_logits=logits_output.next_token_logits,
|
||||
@@ -2844,34 +2863,6 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
bs=bs,
|
||||
)
|
||||
|
||||
if SIMULATE_ACC_LEN > 0:
|
||||
if SIMULATE_ACC_TOKEN_MODE not in ("fixed", "real-draft-token"):
|
||||
raise ValueError(
|
||||
"Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE "
|
||||
f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or "
|
||||
"'real-draft-token'."
|
||||
)
|
||||
|
||||
if SIMULATE_ACC_TOKEN_MODE == "real-draft-token" and target_predict is None:
|
||||
# The sampling-verify branch does not materialize the target argmax.
|
||||
target_predict = torch.argmax(
|
||||
logits_output.next_token_logits, dim=-1
|
||||
).view(bs, int(self.block_size))
|
||||
apply_dflash_simulated_acceptance(
|
||||
candidates=candidates,
|
||||
target_predict=target_predict,
|
||||
accept_len=accept_len,
|
||||
commit_lens=commit_lens,
|
||||
bonus=bonus,
|
||||
out_tokens=out_tokens,
|
||||
simulate_acc_len=SIMULATE_ACC_LEN,
|
||||
simulate_acc_method=SIMULATE_ACC_METHOD,
|
||||
simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE,
|
||||
)
|
||||
# The Triton path may have written new_seq_lens from the real
|
||||
# accept_len; recompute it from the forced commit_lens.
|
||||
new_seq_lens = None
|
||||
|
||||
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
|
||||
sampling_info,
|
||||
next_token_logits=logits_output.next_token_logits,
|
||||
|
||||
@@ -19,11 +19,20 @@ def _domino_gru_cell(
|
||||
)
|
||||
|
||||
|
||||
# Throughput policy for the logical full-vocabulary tensor. TP>1 uses the
|
||||
# compact candidate pool when this gathered representation would be large.
|
||||
# A full-vocab gather is one collective; the candidate pool takes four
|
||||
# all-gathers and an all-reduce. Eager stays launch bound until the gathered
|
||||
# logits are large, while a captured graph pays only GPU time.
|
||||
_DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES = 96 * 1024 * 1024
|
||||
|
||||
|
||||
def _domino_prefers_tp_candidate_pool(
|
||||
*, batch_size: int, full_base_logits_bytes: int, in_cuda_graph: bool
|
||||
) -> bool:
|
||||
if in_cuda_graph:
|
||||
return batch_size > 1
|
||||
return full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES
|
||||
|
||||
|
||||
def _domino_tp_first_ids(
|
||||
local_logits: torch.Tensor,
|
||||
*,
|
||||
@@ -315,7 +324,7 @@ def domino_greedy_rollout(
|
||||
lm_head_org_vocab_start: int = 0,
|
||||
lm_head_num_org: int | None = None,
|
||||
lm_head_num_org_padded: int | None = None,
|
||||
prefer_tp_candidate_pool: bool | None = None,
|
||||
in_cuda_graph: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Generate a Domino chain using one block-shared base-logit candidate pool."""
|
||||
if draft_hidden.ndim != 3:
|
||||
@@ -377,17 +386,17 @@ def domino_greedy_rollout(
|
||||
local_logits = F.linear(logits_input, weight).view(
|
||||
num_proposals, batch_size, local_vocab_size
|
||||
)
|
||||
if prefer_tp_candidate_pool is None:
|
||||
full_base_logits_bytes = (
|
||||
num_proposals * batch_size * int(vocab_size) * local_logits.element_size()
|
||||
)
|
||||
prefer_tp_candidate_pool = (
|
||||
full_base_logits_bytes > _DOMINO_TP_FULL_BASE_LOGITS_MAX_BYTES
|
||||
)
|
||||
use_tp_candidate_pool = (
|
||||
tp_size > 1
|
||||
and prefer_tp_candidate_pool
|
||||
and 0 < candidate_pool_size < int(vocab_size)
|
||||
and _domino_prefers_tp_candidate_pool(
|
||||
batch_size=batch_size,
|
||||
full_base_logits_bytes=num_proposals
|
||||
* batch_size
|
||||
* int(vocab_size)
|
||||
* local_logits.element_size(),
|
||||
in_cuda_graph=in_cuda_graph,
|
||||
)
|
||||
)
|
||||
first_ids = None
|
||||
candidate_ids = None
|
||||
|
||||
@@ -211,6 +211,18 @@ SIMULATE_ACC_LEN = envs.SGLANG_SIMULATE_ACC_LEN.get() # turn off if < 0
|
||||
SIMULATE_ACC_METHOD = envs.SGLANG_SIMULATE_ACC_METHOD.get()
|
||||
SIMULATE_ACC_TOKEN_MODE = envs.SGLANG_SIMULATE_ACC_TOKEN_MODE.get()
|
||||
|
||||
|
||||
def check_simulate_acc_token_mode() -> None:
|
||||
if SIMULATE_ACC_LEN > 0 and SIMULATE_ACC_TOKEN_MODE not in (
|
||||
"fixed",
|
||||
"real-draft-token",
|
||||
):
|
||||
raise ValueError(
|
||||
"Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE "
|
||||
f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or 'real-draft-token'."
|
||||
)
|
||||
|
||||
|
||||
TREE_TRAVERSE_TIME_THRESHOLD = 1 # TODO: set this properly
|
||||
TREE_SPEC_KERNEL_AVAILABLE = (
|
||||
_is_cuda or _is_musa
|
||||
|
||||
Reference in New Issue
Block a user