[qwen 3.8 next] Fuse NEXTN verify and draft graph input preparation (#41175)

This commit is contained in:
Qiaolin Yu
2026-10-01 17:17:34 -07:00
committed by GitHub
parent 2a3655fea1
commit 67eab57057
14 changed files with 1107 additions and 76 deletions
@@ -35,7 +35,7 @@ def _get_oot_speculative_cache_locs_fn() -> Callable[..., torch.Tensor] | None:
return current_platform.get_speculative_cache_locs_fn()
@triton.jit
@triton.jit(do_not_specialize=["BS"])
def assign_draft_cache_locs_contiguous(
req_pool_indices,
req_to_token,
@@ -44,6 +44,11 @@ def assign_draft_cache_locs_contiguous(
pool_len: tl.constexpr,
topk: tl.constexpr,
speculative_num_steps: tl.constexpr,
positions=None,
mrope=None,
BS=0,
WRITE_POSITIONS: tl.constexpr = False,
WRITE_MROPE: tl.constexpr = False,
):
BLOCK_SIZE: tl.constexpr = 128
pid = tl.program_id(axis=0)
@@ -61,6 +66,17 @@ def assign_draft_cache_locs_contiguous(
data = tl.load(token_pool + kv_start + copy_offset, mask=mask)
tl.store(out_cache_ptr + copy_offset, data, mask=mask)
if WRITE_POSITIONS:
topk_offset = tl.arange(0, BLOCK_SIZE)
tl.store(positions + pid * topk + topk_offset, kv_start, topk_offset < topk)
if WRITE_MROPE:
for axis in tl.static_range(3):
tl.store(
mrope + axis * BS * topk + pid * topk + topk_offset,
kv_start,
topk_offset < topk,
)
@triton.jit
def generate_draft_decode_kv_indices(
@@ -89,3 +89,394 @@ def fill_accept_out_cache_loc_func(
accept_out_cache_loc,
next_power_of_2(size),
)
@triton.jit(do_not_specialize=["BS", "PREDICT_SIZE"])
def _verify_commit_outputs_kernel(
predict,
accept_index,
accept_lens,
seq_lens,
new_seq_lens,
bonus,
BS,
PREDICT_SIZE,
PREDICT_STRIDE: tl.constexpr,
INDEX_STRIDE_0: tl.constexpr,
INDEX_STRIDE_1: tl.constexpr,
ACCEPT_STRIDE: tl.constexpr,
SEQ_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
last_steps,
track_steps,
correct,
select,
ids,
DRAFT_WIDTH: tl.constexpr,
TRACK_INTERVAL: tl.constexpr,
):
row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
valid = row < BS
accepted = tl.load(accept_lens + row * ACCEPT_STRIDE, valid, other=1).to(tl.int64)
length = tl.load(seq_lens + row * SEQ_STRIDE, valid, other=0).to(tl.int64)
idx = tl.load(
accept_index + row * INDEX_STRIDE_0 + (accepted - 1) * INDEX_STRIDE_1,
valid,
other=0,
).to(tl.int64)
if DRAFT_WIDTH > 0:
tl.store(last_steps + row, idx - row * DRAFT_WIDTH, valid)
tl.store(correct + row, accepted - 1, valid)
tl.store(select + row, row * DRAFT_WIDTH + accepted - 1, valid)
output_token = tl.load(
predict + row * PREDICT_STRIDE, row < PREDICT_SIZE, other=0
)
tl.store(ids + row, output_token.to(tl.int64), row < PREDICT_SIZE)
if TRACK_INTERVAL > 0:
post = length + accepted
crossing = length // TRACK_INTERVAL != post // TRACK_INTERVAL
tracking_position = tl.maximum(
post // TRACK_INTERVAL * TRACK_INTERVAL - length - 1, 0
)
track = (
tl.load(
accept_index
+ row * INDEX_STRIDE_0
+ tracking_position * INDEX_STRIDE_1,
valid & crossing,
other=0,
).to(tl.int64)
- row * DRAFT_WIDTH
)
tl.store(track_steps + row, tl.where(crossing, track, -1), valid)
idx = tl.where(idx < 0, idx + PREDICT_SIZE, idx)
token = tl.load(predict + idx * PREDICT_STRIDE, valid, other=0)
tl.store(new_seq_lens + row, length + accepted, valid)
tl.store(bonus + row, token, valid)
def prepare_verify_commit_outputs(
predict,
accept_index,
accept_lens,
seq_lens,
num_draft_tokens=0,
mamba_track_interval=0,
):
bs = accept_lens.numel()
new_seq_lens = torch.empty(
(bs,),
device=seq_lens.device,
dtype=torch.promote_types(seq_lens.dtype, accept_lens.dtype),
)
bonus = torch.empty((bs,), device=predict.device, dtype=torch.int32)
last_steps = track_steps = correct = select = ids = None
if num_draft_tokens:
last_steps = torch.empty((bs,), device=seq_lens.device, dtype=torch.int64)
if mamba_track_interval:
track_steps = torch.empty_like(last_steps)
correct = torch.empty_like(accept_lens, memory_format=torch.contiguous_format)
select = torch.empty_like(last_steps)
ids = torch.empty((predict.numel(),), device=predict.device, dtype=torch.int64)
if bs:
size = max(bs, predict.numel()) if num_draft_tokens else bs
_verify_commit_outputs_kernel[(triton.cdiv(size, 128),)](
predict,
accept_index,
accept_lens,
seq_lens,
new_seq_lens,
bonus,
bs,
predict.numel(),
predict.stride(0),
accept_index.stride(0),
accept_index.stride(1),
accept_lens.stride(0),
seq_lens.stride(0),
BLOCK=128,
last_steps=last_steps,
track_steps=track_steps,
correct=correct,
select=select,
ids=ids,
DRAFT_WIDTH=num_draft_tokens,
TRACK_INTERVAL=mamba_track_interval,
)
if num_draft_tokens:
return new_seq_lens, bonus, (last_steps, track_steps), (correct, select, ids)
return new_seq_lens, bonus
@triton.jit(do_not_specialize=["BS", "TOKENS"])
def _draft_extend_inputs_kernel(
accept_lens,
predict,
correct,
select,
ids,
BS,
TOKENS,
WIDTH: tl.constexpr,
ACCEPT_STRIDE: tl.constexpr,
PREDICT_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
accepted = tl.load(accept_lens + offset * ACCEPT_STRIDE, offset < BS, other=1).to(
tl.int64
)
tl.store(correct + offset, accepted - 1, offset < BS)
tl.store(select + offset, offset * WIDTH + accepted - 1, offset < BS)
tokens = tl.load(predict + offset * PREDICT_STRIDE, offset < TOKENS, other=0)
tl.store(ids + offset, tokens.to(tl.int64), offset < TOKENS)
def prepare_draft_extend_inputs(accept_lens, predict, num_draft_tokens):
bs, tokens = accept_lens.numel(), predict.numel()
correct = torch.empty((bs,), device=accept_lens.device, dtype=accept_lens.dtype)
select = torch.empty((bs,), device=accept_lens.device, dtype=torch.int64)
ids = torch.empty((tokens,), device=predict.device, dtype=torch.int64)
if max(bs, tokens):
_draft_extend_inputs_kernel[(triton.cdiv(max(bs, tokens), 128),)](
accept_lens,
predict,
correct,
select,
ids,
bs,
tokens,
num_draft_tokens,
accept_lens.stride(0),
predict.stride(0),
BLOCK=128,
)
return correct, select, ids
@triton.jit(do_not_specialize=["BS"])
def _prepare_draft_extend_lengths_kernel(
seq_lens,
prefix_lens,
extend_lens,
post_lens,
BS,
STRIDE: tl.constexpr,
WIDTH: tl.constexpr,
FRONT: tl.constexpr,
BLOCK: tl.constexpr,
positions=None,
start_locs=None,
mrope=None,
MAKE_POSITIONS: tl.constexpr = False,
MAKE_MROPE: tl.constexpr = False,
):
row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
base = tl.load(seq_lens + row * STRIDE, row < BS, other=0).to(tl.int64)
tl.store(prefix_lens + row, tl.maximum(base - FRONT, 0), row < BS)
tl.store(extend_lens + row, WIDTH + FRONT, row < BS)
tl.store(post_lens + row, base + WIDTH, row < BS)
if MAKE_POSITIONS:
tl.static_assert(FRONT == 0)
tl.store(start_locs + row, row * WIDTH, row < BS)
req = row // WIDTH
token_base = tl.load(seq_lens + req * STRIDE, row < BS * WIDTH, other=0).to(
tl.int64
)
prefix = tl.maximum(token_base - FRONT, 0).to(tl.int32)
position = prefix + (row % WIDTH).to(tl.int32)
tl.store(positions + row, position.to(tl.int64), row < BS * WIDTH)
if MAKE_MROPE:
for axis in tl.static_range(3):
tl.store(
mrope + axis * BS * WIDTH + row,
position.to(tl.int64),
row < BS * WIDTH,
)
def prepare_draft_extend_lengths(seq_lens, num_draft_tokens, front_offset):
bs = seq_lens.numel()
prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device)
extend = torch.empty_like(prefix)
post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device)
if bs:
_prepare_draft_extend_lengths_kernel[(triton.cdiv(bs, 128),)](
seq_lens,
prefix,
extend,
post,
bs,
seq_lens.stride(0),
num_draft_tokens,
front_offset,
BLOCK=128,
)
return prefix, extend, post
def prepare_draft_extend_layout(seq_lens, num_draft_tokens, with_mrope):
bs = seq_lens.numel()
prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device)
extend = torch.empty_like(prefix)
post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device)
positions = torch.empty(
(bs * num_draft_tokens,), dtype=torch.int64, device=seq_lens.device
)
starts = torch.empty_like(prefix)
mrope = (
torch.empty(
(3, bs * num_draft_tokens), dtype=torch.int64, device=seq_lens.device
)
if with_mrope
else None
)
if bs:
_prepare_draft_extend_lengths_kernel[
(triton.cdiv(bs * num_draft_tokens, 128),)
](
seq_lens,
prefix,
extend,
post,
bs,
seq_lens.stride(0),
num_draft_tokens,
0,
BLOCK=128,
positions=positions,
start_locs=starts,
mrope=mrope,
MAKE_POSITIONS=True,
MAKE_MROPE=with_mrope,
)
return prefix, extend, post, (positions, starts, mrope)
@triton.jit(do_not_specialize=["BS"])
def _build_chain_tree_kernel(
bonus,
draft,
seq,
mask,
positions,
retrieve,
tokens,
BS,
WIDTH: tl.constexpr,
BONUS_STRIDE: tl.constexpr,
DRAFT_ROW: tl.constexpr,
DRAFT_COL: tl.constexpr,
SEQ_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
req_indices,
req_to_token,
out_cache_loc,
mrope,
REQ_STRIDE: tl.constexpr,
TABLE_ROW: tl.constexpr,
TABLE_COL: tl.constexpr,
WRITE_CACHE: tl.constexpr,
WRITE_MROPE: tl.constexpr,
):
offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
row = offset // WIDTH
col = offset % WIDTH
valid = offset < BS * WIDTH
first = tl.load(bonus + row * BONUS_STRIDE, valid & (col == 0), other=0)
rest = tl.load(
draft + row * DRAFT_ROW + (col - 1) * DRAFT_COL, valid & (col > 0), other=0
)
tl.store(tokens + offset, tl.where(col == 0, first, rest), valid)
length = tl.load(seq + row * SEQ_STRIDE, valid, other=0).to(tl.int64)
tl.store(positions + offset, length + col, valid)
if WRITE_CACHE:
req = tl.load(req_indices + row * REQ_STRIDE, valid, other=0).to(tl.int64)
loc = tl.load(
req_to_token + req * TABLE_ROW + (length + col) * TABLE_COL, valid, other=0
)
tl.store(out_cache_loc + offset, loc, valid)
if WRITE_MROPE:
for axis in tl.static_range(3):
tl.store(mrope + axis * BS * WIDTH + offset, length + col, valid)
tl.store(retrieve + offset, offset, valid)
tl.store(
retrieve + BS * WIDTH + offset, tl.where(col + 1 < WIDTH, col + 1, -1), valid
)
tl.store(retrieve + 2 * BS * WIDTH + offset, -1, valid)
mask_col = offset % WIDTH
mask_row = (offset // WIDTH) % WIDTH
tl.store(mask + offset, mask_col <= mask_row, offset < BS * WIDTH * WIDTH)
def build_chain_tree(
bonus_tokens,
draft_tokens,
seq_lens,
tree_mask_buf=None,
req_pool_indices=None,
req_to_token=None,
with_mrope=False,
):
bs, depth = draft_tokens.shape
width = depth + 1
assert bonus_tokens.numel() == seq_lens.numel() == bs
mask = tree_mask_buf
if mask is None:
mask = torch.empty(
(bs * width * width,), dtype=torch.bool, device=seq_lens.device
)
assert mask.is_contiguous() and mask.numel() >= bs * width * width
positions = torch.empty((bs * width,), dtype=torch.int64, device=seq_lens.device)
retrieve = torch.empty((3, bs, width), dtype=torch.int64, device=seq_lens.device)
tokens = torch.empty(
(bs * width,),
dtype=torch.promote_types(bonus_tokens.dtype, draft_tokens.dtype),
device=seq_lens.device,
)
out_cache_loc = torch.empty_like(positions) if req_to_token is not None else None
mrope = (
torch.empty((3, bs * width), dtype=torch.int64, device=seq_lens.device)
if with_mrope
else None
)
if req_to_token is not None:
assert req_pool_indices is not None and req_pool_indices.numel() == bs
if bs:
_build_chain_tree_kernel[(triton.cdiv(bs * width * width, 128),)](
bonus_tokens,
draft_tokens,
seq_lens,
mask,
positions,
retrieve,
tokens,
bs,
width,
bonus_tokens.stride(0),
draft_tokens.stride(0),
draft_tokens.stride(1),
seq_lens.stride(0),
BLOCK=128,
req_indices=req_pool_indices,
req_to_token=req_to_token,
out_cache_loc=out_cache_loc,
mrope=mrope,
REQ_STRIDE=(
req_pool_indices.stride(0) if req_pool_indices is not None else 0
),
TABLE_ROW=req_to_token.stride(0) if req_to_token is not None else 0,
TABLE_COL=req_to_token.stride(1) if req_to_token is not None else 0,
WRITE_CACHE=req_to_token is not None,
WRITE_MROPE=with_mrope,
)
return (
mask,
positions,
retrieve[0],
retrieve[1],
retrieve[2],
tokens,
out_cache_loc,
mrope,
)
@@ -41,6 +41,7 @@ from sglang.srt.layers.attention.qsa.sparse_attn import (
sparse_gqa_fwd_interface_triton_ck,
sparse_gqa_packed_decode_triton,
)
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.layers.cp.utils import (
ContextParallelStrategyKind,
cp_materialize_global_token_order,
@@ -50,6 +51,7 @@ from sglang.srt.layers.cp.utils import (
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.utils import is_hip
logger = logging.getLogger(__name__)
@@ -252,6 +254,7 @@ class QwenSparseAttnBackend(AttentionBackend):
self._trtllm_counters = {}
self._graph_extend_lens = None
self._graph_extend_lens_pin = None
self._verify_mask = None
@staticmethod
def _is_speculative_paged_mode(forward_mode) -> bool:
@@ -795,11 +798,24 @@ class QwenSparseAttnBackend(AttentionBackend):
num_padding=num_padding if num_padding is not None else 0,
)
@property
def verify_mask(self) -> Optional[VerifyMask]:
return self._verify_mask
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int) -> None:
if self.device is None:
raise RuntimeError(
"QSA backend requires a ModelRunner to initialize CUDA graph state"
)
self._verify_mask = maybe_create_verify_mask(
is_draft_runner=self.runner is not None and self.runner.is_draft_worker,
skip_prefill=False,
max_bs=max_bs,
max_context_len=self.max_context_len,
num_draft_tokens=get_spec().speculative_num_draft_tokens,
device=self.device,
is_read=False,
)
self._cuda_graph_max_tokens = max_num_tokens
max_blocks = math.ceil(self.max_context_len / self.compress_ratio)
max_pages = max(
+3
View File
@@ -122,6 +122,9 @@ class GenerationBatchResult:
# Every stage holds the KV for its own layers, so every stage has to compact
# that path into its committed prefix; only the last stage can compute it.
accept_index: Optional[torch.Tensor] = None
prepared_draft_extend_inputs: Optional[
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
] = None
# Refs the worker wants scheduler to keep alive for the same 2-iter window
# as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec
@@ -934,6 +934,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
*,
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
return_hidden_states_before_norm: bool,
extend_position_info=None,
spec_mrope_positions: Optional[torch.Tensor] = None,
):
# init_new must not mutate the input ScheduleBatch; per-forward
# overrides go through explicit keyword arguments.
@@ -1157,12 +1159,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ret.extend_seq_lens = extend_seq_lens
ret.extend_prefix_lens = extend_prefix_lens
ret.extend_num_tokens = batch.extend_num_tokens
positions, ret.extend_start_loc = compute_position(
model_runner.prefill_attention_backend_str,
ret.extend_prefix_lens,
ret.extend_seq_lens,
ret.extend_num_tokens,
)
if extend_position_info is None:
positions, ret.extend_start_loc = compute_position(
model_runner.prefill_attention_backend_str,
ret.extend_prefix_lens,
ret.extend_seq_lens,
ret.extend_num_tokens,
)
else:
positions, ret.extend_start_loc, _ = extend_position_info
if ret.positions is None:
ret.positions = positions
ret.extend_logprob_start_lens_cpu = extend_logprob_start_lens
@@ -1171,7 +1176,13 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ret._init_ngram_embedding_info(batch, device)
if model_runner.model_config.model_is_mrope:
if (
if spec_mrope_positions is not None:
ret.mrope_positions = spec_mrope_positions
elif (
extend_position_info is not None and extend_position_info[2] is not None
):
ret.mrope_positions = extend_position_info[2]
elif (
ret.spec_info is not None
and getattr(ret.spec_info, "positions", None) is not None
):
@@ -1400,9 +1411,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
seq_positions = seq_positions.view(batch_size, -1)
# Split text-only and mixed batches here because SpecV2 text-only batches can avoid an extra D2H.
if all(mm_input is None for mm_input in mm_inputs):
mrope_delta_tensor = torch.zeros(
(batch_size, 1), dtype=torch.int64, device=device
self.mrope_positions = (
seq_positions.to(torch.int64).flatten().unsqueeze(0).repeat(3, 1)
)
return
else:
mrope_deltas = [
(
@@ -624,7 +624,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch.req_pool_indices,
]
if self.model_runner.model_config.model_is_mrope:
buffers.mrope_positions.zero_()
if bs != raw_bs or forward_batch.mrope_positions is None:
buffers.mrope_positions.zero_()
if forward_batch.mrope_positions is not None:
copy_dsts.append(buffers.mrope_positions[:, :raw_num_token])
copy_srcs.append(forward_batch.mrope_positions)
@@ -552,7 +552,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch.req_pool_indices,
]
if self.model_runner.model_config.model_is_mrope:
buffers.mrope_positions.zero_()
if (
bs * self.captured_req_width != num_tokens
or forward_batch.mrope_positions is None
):
buffers.mrope_positions.zero_()
if forward_batch.mrope_positions is not None:
copy_dsts.append(buffers.mrope_positions[:, :num_tokens])
copy_srcs.append(forward_batch.mrope_positions)
@@ -32,6 +32,8 @@ class EagleVerifyInput(SpecInput):
# Stacked per-step draft proposal distribution q, shape (bs, num_steps,
# vocab); only set under rejection sampling. Consumed by the verify kernel.
draft_probs: torch.Tensor = None
prepared_out_cache_loc: Optional[torch.Tensor] = None
prepared_mrope_positions: Optional[torch.Tensor] = None
# Shape info for padding
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
+12 -8
View File
@@ -566,14 +566,17 @@ def eagle_prepare_for_verify(
# Uniform variant: end offsets (= start + draft_token_num) are computed
# inside the kernel, keeping the eager `seq_lens + N` add off the host
# critical path (bs=1 MTP inter-phase seam).
batch.out_cache_loc = assign_extend_cache_locs_uniform_func(
req_pool_indices=batch.req_pool_indices,
req_to_token=req_to_token_pool.req_to_token,
start_offset=batch.seq_lens,
batch_size=bs,
draft_token_num=verify_input.draft_token_num,
device=device,
)
if verify_input.prepared_out_cache_loc is not None:
batch.out_cache_loc = verify_input.prepared_out_cache_loc
else:
batch.out_cache_loc = assign_extend_cache_locs_uniform_func(
req_pool_indices=batch.req_pool_indices,
req_to_token=req_to_token_pool.req_to_token,
start_offset=batch.seq_lens,
batch_size=bs,
draft_token_num=verify_input.draft_token_num,
device=device,
)
batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle(
batch, verify_input.draft_token_num
@@ -601,6 +604,7 @@ def eagle_prepare_for_verify(
target_worker.model_runner,
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=verify_input.prepared_mrope_positions,
)
# Run attention backend plan and cuda graph preparation
@@ -7,7 +7,13 @@ import torch
from sglang.kernels.ops.speculative.cache_locs import (
assign_draft_cache_locs_contiguous,
)
from sglang.kernels.ops.speculative.eagle import fill_bonus_tokens_func
from sglang.kernels.ops.speculative.eagle import (
build_chain_tree,
fill_bonus_tokens_func,
prepare_draft_extend_layout,
prepare_draft_extend_lengths,
prepare_verify_commit_outputs,
)
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.model_executor.forward_batch_info import (
@@ -16,6 +22,7 @@ from sglang.srt.model_executor.forward_batch_info import (
ForwardMode,
PPProxyTensors,
)
from sglang.srt.runtime_context import mamba_track_grid
from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput
from sglang.srt.speculative.eagle_utils import (
TreeMaskMode,
@@ -155,7 +162,22 @@ def prepare_for_draft_extend(
)
# init_new requires both list or both Tensor;
# gpu_only emits device tensors to skip H2D.
if gpu_only:
fused_lengths = gpu_only and batch.seq_lens.is_cuda
extend_position_info = None
if fused_lengths and not widen and draft_extend_input.positions is None:
batch.prefix_lens, batch.extend_lens, post_seq_lens, extend_position_info = (
prepare_draft_extend_layout(
batch.seq_lens,
num_draft_tokens,
draft_model_runner.model_config.model_is_mrope
and all(x is None for x in batch.multimodal_inputs),
)
)
elif fused_lengths:
batch.prefix_lens, batch.extend_lens, post_seq_lens = (
prepare_draft_extend_lengths(batch.seq_lens, num_draft_tokens, front_offset)
)
elif gpu_only:
batch.prefix_lens = (batch.seq_lens - front_offset).clamp(min=0).to(torch.int32)
batch.extend_lens = torch.full(
(bs,), num_window_tokens, dtype=torch.int32, device=batch.seq_lens.device
@@ -181,10 +203,13 @@ def prepare_for_draft_extend(
draft_model_runner,
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=return_hidden_states_before_norm,
extend_position_info=extend_position_info,
)
# Forward sees post-write length (draft extend writes num_draft_tokens
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
forward_batch.seq_lens = forward_batch.seq_lens + num_draft_tokens
forward_batch.seq_lens = (
post_seq_lens if fused_lengths else forward_batch.seq_lens + num_draft_tokens
)
if not gpu_only:
forward_batch.seq_lens_cpu = forward_batch.seq_lens_cpu + num_draft_tokens
forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum())
@@ -220,6 +245,8 @@ def prepare_for_draft(
num_steps: int,
):
prepared_positions = None
prepared_mrope_positions = None
if not batch.forward_mode.is_idle():
bs = len(batch.seq_lens)
@@ -242,6 +269,14 @@ def prepare_for_draft(
num_steps,
)
else:
if batch.seq_lens.is_cuda and topk == 1:
prepared_positions = torch.empty_like(batch.seq_lens)
if draft_model_runner.model_config.model_is_mrope and all(
x is None for x in batch.multimodal_inputs
):
prepared_mrope_positions = torch.empty(
(3, bs), dtype=torch.int64, device=batch.device
)
# FIXME(lsyin): align with the default code path
assign_draft_cache_locs_contiguous[(bs,)](
batch.req_pool_indices,
@@ -251,6 +286,11 @@ def prepare_for_draft(
req_to_token_pool.req_to_token.shape[1],
topk,
num_steps,
positions=prepared_positions,
mrope=prepared_mrope_positions,
BS=bs,
WRITE_POSITIONS=prepared_positions is not None,
WRITE_MROPE=prepared_mrope_positions is not None,
)
else:
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
@@ -301,12 +341,17 @@ def prepare_for_draft(
if draft_model_runner.spec_algorithm.is_standalone()
else CaptureHiddenMode.LAST
)
draft_input.positions = batch.seq_lens.repeat_interleave(topk, dim=0)
draft_input.positions = (
prepared_positions
if prepared_positions is not None
else batch.seq_lens.repeat_interleave(topk, dim=0)
)
forward_batch = ForwardBatch.init_new(
batch,
draft_model_runner,
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=prepared_mrope_positions,
)
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
forward_batch
@@ -364,6 +409,42 @@ def build_eagle_verify_input(
else:
seq_lens_sum = bs * target_attn_backend.max_context_len
prepared_out_cache_loc = None
prepared_mrope_positions = None
if (
batch.seq_lens.is_cuda
and topk == 1
and num_draft_tokens == num_steps + 1
and draft_tokens.shape == (bs, num_steps)
and mask_mode == TreeMaskMode.QLEN_ONLY
):
tree_outputs = build_chain_tree(
draft_input.bonus_tokens,
draft_tokens,
batch.seq_lens,
tree_mask_buf,
req_pool_indices=batch.req_pool_indices,
req_to_token=batch.req_to_token_pool.req_to_token,
with_mrope=target_worker.model_runner.model_config.model_is_mrope
and all(x is None for x in batch.multimodal_inputs),
)
prepared_out_cache_loc, prepared_mrope_positions = tree_outputs[6:]
tree_outputs = tree_outputs[:6]
else:
tree_outputs = build_tree_kernel_efficient(
draft_input.bonus_tokens,
parent_list,
top_scores_index,
draft_tokens,
batch.seq_lens,
seq_lens_sum,
topk,
num_steps,
num_draft_tokens,
mask_mode,
tree_mask_buf,
fill_prefix_mask=fill_mask,
)
(
tree_mask,
position,
@@ -371,20 +452,7 @@ def build_eagle_verify_input(
retrieve_next_token,
retrieve_next_sibling,
draft_tokens,
) = build_tree_kernel_efficient(
draft_input.bonus_tokens,
parent_list,
top_scores_index,
draft_tokens,
batch.seq_lens,
seq_lens_sum,
topk,
num_steps,
num_draft_tokens,
mask_mode,
tree_mask_buf,
fill_prefix_mask=fill_mask,
)
) = tree_outputs
return EagleVerifyInput(
draft_token=draft_tokens,
@@ -401,6 +469,8 @@ def build_eagle_verify_input(
seq_lens_sum=None,
seq_lens_cpu=None,
draft_probs=draft_probs,
prepared_out_cache_loc=prepared_out_cache_loc,
prepared_mrope_positions=prepared_mrope_positions,
)
@@ -599,7 +669,30 @@ def run_eagle_verify(
grammar_mask,
uno_target_max_top_k=uno_target_max_top_k,
)
new_seq_lens = batch.seq_lens + accept_lens
fused_commit_outputs = predict.is_cuda and not batch.forward_mode.is_idle()
prepared_commit_steps = prepared_draft_inputs = None
if fused_commit_outputs and topk == 1:
track_interval = (
mamba_track_grid(batch.tree_cache.page_size)
if batch.mamba_track_indices is not None
else 0
)
new_seq_lens, bonus_tokens, prepared_commit_steps, prepared_draft_inputs = (
prepare_verify_commit_outputs(
predict,
accept_index,
accept_lens,
batch.seq_lens,
num_draft_tokens=num_draft_tokens,
mamba_track_interval=track_interval,
)
)
elif fused_commit_outputs:
new_seq_lens, bonus_tokens = prepare_verify_commit_outputs(
predict, accept_index, accept_lens, batch.seq_lens
)
else:
new_seq_lens = batch.seq_lens + accept_lens
clear_unaccepted_c128 = getattr(
token_to_kv_pool_allocator.get_kvcache(),
"clear_unaccepted_c128_draft_states",
@@ -620,9 +713,10 @@ def run_eagle_verify(
accept_lens,
accept_index,
num_draft_tokens,
prepared_step_indices=prepared_commit_steps,
)
if not batch.forward_mode.is_idle():
if not fused_commit_outputs and not batch.forward_mode.is_idle():
accept_tokens = predict[accept_index]
bonus_tokens = torch.empty_like(accept_lens, dtype=torch.int32)
# stride = accept_tokens per-req width = accept_index.shape[1]
@@ -634,7 +728,7 @@ def run_eagle_verify(
accept_index.shape[1],
bs,
)
else:
elif batch.forward_mode.is_idle():
bonus_tokens = torch.empty((0,), device=device, dtype=torch.int32)
if batch.return_logprob and not batch.forward_mode.is_idle():
@@ -672,4 +766,5 @@ def run_eagle_verify(
routed_experts_output=forward_batch_output.routed_experts_output,
indexer_topk_output=forward_batch_output.indexer_topk_output,
extra_keep_alive_refs=[verify_forward_batch],
prepared_draft_extend_inputs=prepared_draft_inputs,
)
@@ -1028,32 +1028,48 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def _draft_extend_for_decode(
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
):
# Cast to int64 before entering plan stream to avoid cross-stream
# synchronization issues with .to() inside the plan stream context.
if batch_result.prepared_draft_extend_inputs is not None:
num_correct_drafts, select_index, next_token_ids = (
batch_result.prepared_draft_extend_inputs
)
elif batch_result.accept_lens.is_cuda:
from sglang.kernels.ops.speculative.eagle import prepare_draft_extend_inputs
num_correct_drafts, select_index, next_token_ids = (
prepare_draft_extend_inputs(
batch_result.accept_lens,
batch_result.next_token_ids,
self.speculative_num_draft_tokens,
)
)
else:
num_correct_drafts = batch_result.accept_lens - 1
select_index = (
torch.arange(
0,
len(batch.seq_lens) * self.speculative_num_draft_tokens,
self.speculative_num_draft_tokens,
device=self.device,
)
+ batch_result.accept_lens
- 1
)
next_token_ids = batch_result.next_token_ids.to(torch.int64)
# Batch 2: Draft extend
draft_extend_input = EagleDraftExtendInput(
hidden_states=batch_result.logits_output.hidden_states,
# accept_lens includes the bonus token; correct drafts exclude it.
num_correct_drafts=batch_result.accept_lens - 1,
num_correct_drafts=num_correct_drafts,
num_accept_tokens=batch_result.accept_lens,
# Draft-extend fills the whole tree width (num_draft_tokens) per req,
# not num_steps + 1, so DP MLP-sync padding stays consistent for topk > 1.
num_tokens_per_req=self.speculative_num_draft_tokens,
num_tokens_for_logprob_per_req=self.speculative_num_draft_tokens,
)
select_index = (
torch.arange(
0,
len(batch.seq_lens) * self.speculative_num_draft_tokens,
self.speculative_num_draft_tokens,
device=self.device,
)
+ batch_result.accept_lens
- 1
)
# Cast to int64 before entering plan stream to avoid cross-stream
# synchronization issues with .to() inside the plan stream context.
next_token_ids = batch_result.next_token_ids.to(torch.int64)
# Prepare for draft extend in a separate stream
if self.plan_stream:
self.plan_stream.wait_stream(
+36 -18
View File
@@ -313,6 +313,8 @@ def record_stream_for_v2_verify(batch, verify_input, fwd_stream):
"draft_token",
"custom_mask",
"positions",
"prepared_out_cache_loc",
"prepared_mrope_positions",
"retrieve_index",
"retrieve_next_token",
"retrieve_next_sibling",
@@ -932,6 +934,7 @@ def commit_mamba_states_after_verify(
accept_lens: torch.Tensor,
accept_index: torch.Tensor,
draft_token_num: int,
prepared_step_indices: Optional[Tuple[torch.Tensor, Optional[torch.Tensor]]] = None,
) -> None:
"""Commit accepted per-step mamba states into the persistent caches.
@@ -979,12 +982,17 @@ def commit_mamba_states_after_verify(
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
if prepared_step_indices is None:
last_correct_step_indices, mamba_steps_to_track = (
_verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
)
else:
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
commit_gdn_replayssm_fold_after_verify(
spec_state=spec_state,
state_batch_indices=state_batch_indices,
@@ -1015,12 +1023,17 @@ def commit_mamba_states_after_verify(
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
replay_indices = batch.req_pool_indices
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
if prepared_step_indices is None:
last_correct_step_indices, mamba_steps_to_track = (
_verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
)
else:
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
# Advance the per-request circular cursors by the accepted count (incl. the
# bonus token). max_cache_len = ring length L = replayssm_d.shape[-2].
commit_gdn_replayssm_spec(
@@ -1137,12 +1150,17 @@ def commit_mamba_states_after_verify(
# `accept_lens` already includes the bonus token (drafts + 1 per req).
if not batch.forward_mode.is_idle() and accept_index.numel() > 0:
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
if prepared_step_indices is None:
last_correct_step_indices, mamba_steps_to_track = (
_verify_commit_step_indices(
batch=batch,
accept_index=accept_index,
accept_lens=accept_lens,
draft_token_num=draft_token_num,
)
)
else:
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
if hasattr(attn_backend, "update_mamba_state_after_mtp_verify"):
attn_backend.update_mamba_state_after_mtp_verify(