[qwen 3.8 next] Fuse NEXTN verify and draft graph input preparation (#41175)
This commit is contained in:
@@ -35,7 +35,7 @@ def _get_oot_speculative_cache_locs_fn() -> Callable[..., torch.Tensor] | None:
|
||||
return current_platform.get_speculative_cache_locs_fn()
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["BS"])
|
||||
def assign_draft_cache_locs_contiguous(
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
@@ -44,6 +44,11 @@ def assign_draft_cache_locs_contiguous(
|
||||
pool_len: tl.constexpr,
|
||||
topk: tl.constexpr,
|
||||
speculative_num_steps: tl.constexpr,
|
||||
positions=None,
|
||||
mrope=None,
|
||||
BS=0,
|
||||
WRITE_POSITIONS: tl.constexpr = False,
|
||||
WRITE_MROPE: tl.constexpr = False,
|
||||
):
|
||||
BLOCK_SIZE: tl.constexpr = 128
|
||||
pid = tl.program_id(axis=0)
|
||||
@@ -61,6 +66,17 @@ def assign_draft_cache_locs_contiguous(
|
||||
data = tl.load(token_pool + kv_start + copy_offset, mask=mask)
|
||||
tl.store(out_cache_ptr + copy_offset, data, mask=mask)
|
||||
|
||||
if WRITE_POSITIONS:
|
||||
topk_offset = tl.arange(0, BLOCK_SIZE)
|
||||
tl.store(positions + pid * topk + topk_offset, kv_start, topk_offset < topk)
|
||||
if WRITE_MROPE:
|
||||
for axis in tl.static_range(3):
|
||||
tl.store(
|
||||
mrope + axis * BS * topk + pid * topk + topk_offset,
|
||||
kv_start,
|
||||
topk_offset < topk,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def generate_draft_decode_kv_indices(
|
||||
|
||||
@@ -89,3 +89,394 @@ def fill_accept_out_cache_loc_func(
|
||||
accept_out_cache_loc,
|
||||
next_power_of_2(size),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["BS", "PREDICT_SIZE"])
|
||||
def _verify_commit_outputs_kernel(
|
||||
predict,
|
||||
accept_index,
|
||||
accept_lens,
|
||||
seq_lens,
|
||||
new_seq_lens,
|
||||
bonus,
|
||||
BS,
|
||||
PREDICT_SIZE,
|
||||
PREDICT_STRIDE: tl.constexpr,
|
||||
INDEX_STRIDE_0: tl.constexpr,
|
||||
INDEX_STRIDE_1: tl.constexpr,
|
||||
ACCEPT_STRIDE: tl.constexpr,
|
||||
SEQ_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
last_steps,
|
||||
track_steps,
|
||||
correct,
|
||||
select,
|
||||
ids,
|
||||
DRAFT_WIDTH: tl.constexpr,
|
||||
TRACK_INTERVAL: tl.constexpr,
|
||||
):
|
||||
row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
|
||||
valid = row < BS
|
||||
accepted = tl.load(accept_lens + row * ACCEPT_STRIDE, valid, other=1).to(tl.int64)
|
||||
length = tl.load(seq_lens + row * SEQ_STRIDE, valid, other=0).to(tl.int64)
|
||||
idx = tl.load(
|
||||
accept_index + row * INDEX_STRIDE_0 + (accepted - 1) * INDEX_STRIDE_1,
|
||||
valid,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
if DRAFT_WIDTH > 0:
|
||||
tl.store(last_steps + row, idx - row * DRAFT_WIDTH, valid)
|
||||
tl.store(correct + row, accepted - 1, valid)
|
||||
tl.store(select + row, row * DRAFT_WIDTH + accepted - 1, valid)
|
||||
output_token = tl.load(
|
||||
predict + row * PREDICT_STRIDE, row < PREDICT_SIZE, other=0
|
||||
)
|
||||
tl.store(ids + row, output_token.to(tl.int64), row < PREDICT_SIZE)
|
||||
if TRACK_INTERVAL > 0:
|
||||
post = length + accepted
|
||||
crossing = length // TRACK_INTERVAL != post // TRACK_INTERVAL
|
||||
tracking_position = tl.maximum(
|
||||
post // TRACK_INTERVAL * TRACK_INTERVAL - length - 1, 0
|
||||
)
|
||||
track = (
|
||||
tl.load(
|
||||
accept_index
|
||||
+ row * INDEX_STRIDE_0
|
||||
+ tracking_position * INDEX_STRIDE_1,
|
||||
valid & crossing,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
- row * DRAFT_WIDTH
|
||||
)
|
||||
tl.store(track_steps + row, tl.where(crossing, track, -1), valid)
|
||||
idx = tl.where(idx < 0, idx + PREDICT_SIZE, idx)
|
||||
token = tl.load(predict + idx * PREDICT_STRIDE, valid, other=0)
|
||||
tl.store(new_seq_lens + row, length + accepted, valid)
|
||||
tl.store(bonus + row, token, valid)
|
||||
|
||||
|
||||
def prepare_verify_commit_outputs(
|
||||
predict,
|
||||
accept_index,
|
||||
accept_lens,
|
||||
seq_lens,
|
||||
num_draft_tokens=0,
|
||||
mamba_track_interval=0,
|
||||
):
|
||||
bs = accept_lens.numel()
|
||||
new_seq_lens = torch.empty(
|
||||
(bs,),
|
||||
device=seq_lens.device,
|
||||
dtype=torch.promote_types(seq_lens.dtype, accept_lens.dtype),
|
||||
)
|
||||
bonus = torch.empty((bs,), device=predict.device, dtype=torch.int32)
|
||||
last_steps = track_steps = correct = select = ids = None
|
||||
if num_draft_tokens:
|
||||
last_steps = torch.empty((bs,), device=seq_lens.device, dtype=torch.int64)
|
||||
if mamba_track_interval:
|
||||
track_steps = torch.empty_like(last_steps)
|
||||
correct = torch.empty_like(accept_lens, memory_format=torch.contiguous_format)
|
||||
select = torch.empty_like(last_steps)
|
||||
ids = torch.empty((predict.numel(),), device=predict.device, dtype=torch.int64)
|
||||
if bs:
|
||||
size = max(bs, predict.numel()) if num_draft_tokens else bs
|
||||
_verify_commit_outputs_kernel[(triton.cdiv(size, 128),)](
|
||||
predict,
|
||||
accept_index,
|
||||
accept_lens,
|
||||
seq_lens,
|
||||
new_seq_lens,
|
||||
bonus,
|
||||
bs,
|
||||
predict.numel(),
|
||||
predict.stride(0),
|
||||
accept_index.stride(0),
|
||||
accept_index.stride(1),
|
||||
accept_lens.stride(0),
|
||||
seq_lens.stride(0),
|
||||
BLOCK=128,
|
||||
last_steps=last_steps,
|
||||
track_steps=track_steps,
|
||||
correct=correct,
|
||||
select=select,
|
||||
ids=ids,
|
||||
DRAFT_WIDTH=num_draft_tokens,
|
||||
TRACK_INTERVAL=mamba_track_interval,
|
||||
)
|
||||
if num_draft_tokens:
|
||||
return new_seq_lens, bonus, (last_steps, track_steps), (correct, select, ids)
|
||||
return new_seq_lens, bonus
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["BS", "TOKENS"])
|
||||
def _draft_extend_inputs_kernel(
|
||||
accept_lens,
|
||||
predict,
|
||||
correct,
|
||||
select,
|
||||
ids,
|
||||
BS,
|
||||
TOKENS,
|
||||
WIDTH: tl.constexpr,
|
||||
ACCEPT_STRIDE: tl.constexpr,
|
||||
PREDICT_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
|
||||
accepted = tl.load(accept_lens + offset * ACCEPT_STRIDE, offset < BS, other=1).to(
|
||||
tl.int64
|
||||
)
|
||||
tl.store(correct + offset, accepted - 1, offset < BS)
|
||||
tl.store(select + offset, offset * WIDTH + accepted - 1, offset < BS)
|
||||
tokens = tl.load(predict + offset * PREDICT_STRIDE, offset < TOKENS, other=0)
|
||||
tl.store(ids + offset, tokens.to(tl.int64), offset < TOKENS)
|
||||
|
||||
|
||||
def prepare_draft_extend_inputs(accept_lens, predict, num_draft_tokens):
|
||||
bs, tokens = accept_lens.numel(), predict.numel()
|
||||
correct = torch.empty((bs,), device=accept_lens.device, dtype=accept_lens.dtype)
|
||||
select = torch.empty((bs,), device=accept_lens.device, dtype=torch.int64)
|
||||
ids = torch.empty((tokens,), device=predict.device, dtype=torch.int64)
|
||||
if max(bs, tokens):
|
||||
_draft_extend_inputs_kernel[(triton.cdiv(max(bs, tokens), 128),)](
|
||||
accept_lens,
|
||||
predict,
|
||||
correct,
|
||||
select,
|
||||
ids,
|
||||
bs,
|
||||
tokens,
|
||||
num_draft_tokens,
|
||||
accept_lens.stride(0),
|
||||
predict.stride(0),
|
||||
BLOCK=128,
|
||||
)
|
||||
return correct, select, ids
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["BS"])
|
||||
def _prepare_draft_extend_lengths_kernel(
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
extend_lens,
|
||||
post_lens,
|
||||
BS,
|
||||
STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
FRONT: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
positions=None,
|
||||
start_locs=None,
|
||||
mrope=None,
|
||||
MAKE_POSITIONS: tl.constexpr = False,
|
||||
MAKE_MROPE: tl.constexpr = False,
|
||||
):
|
||||
row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
|
||||
base = tl.load(seq_lens + row * STRIDE, row < BS, other=0).to(tl.int64)
|
||||
tl.store(prefix_lens + row, tl.maximum(base - FRONT, 0), row < BS)
|
||||
tl.store(extend_lens + row, WIDTH + FRONT, row < BS)
|
||||
tl.store(post_lens + row, base + WIDTH, row < BS)
|
||||
if MAKE_POSITIONS:
|
||||
tl.static_assert(FRONT == 0)
|
||||
tl.store(start_locs + row, row * WIDTH, row < BS)
|
||||
req = row // WIDTH
|
||||
token_base = tl.load(seq_lens + req * STRIDE, row < BS * WIDTH, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
prefix = tl.maximum(token_base - FRONT, 0).to(tl.int32)
|
||||
position = prefix + (row % WIDTH).to(tl.int32)
|
||||
tl.store(positions + row, position.to(tl.int64), row < BS * WIDTH)
|
||||
if MAKE_MROPE:
|
||||
for axis in tl.static_range(3):
|
||||
tl.store(
|
||||
mrope + axis * BS * WIDTH + row,
|
||||
position.to(tl.int64),
|
||||
row < BS * WIDTH,
|
||||
)
|
||||
|
||||
|
||||
def prepare_draft_extend_lengths(seq_lens, num_draft_tokens, front_offset):
|
||||
bs = seq_lens.numel()
|
||||
prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device)
|
||||
extend = torch.empty_like(prefix)
|
||||
post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device)
|
||||
if bs:
|
||||
_prepare_draft_extend_lengths_kernel[(triton.cdiv(bs, 128),)](
|
||||
seq_lens,
|
||||
prefix,
|
||||
extend,
|
||||
post,
|
||||
bs,
|
||||
seq_lens.stride(0),
|
||||
num_draft_tokens,
|
||||
front_offset,
|
||||
BLOCK=128,
|
||||
)
|
||||
return prefix, extend, post
|
||||
|
||||
|
||||
def prepare_draft_extend_layout(seq_lens, num_draft_tokens, with_mrope):
|
||||
bs = seq_lens.numel()
|
||||
prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device)
|
||||
extend = torch.empty_like(prefix)
|
||||
post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device)
|
||||
positions = torch.empty(
|
||||
(bs * num_draft_tokens,), dtype=torch.int64, device=seq_lens.device
|
||||
)
|
||||
starts = torch.empty_like(prefix)
|
||||
mrope = (
|
||||
torch.empty(
|
||||
(3, bs * num_draft_tokens), dtype=torch.int64, device=seq_lens.device
|
||||
)
|
||||
if with_mrope
|
||||
else None
|
||||
)
|
||||
if bs:
|
||||
_prepare_draft_extend_lengths_kernel[
|
||||
(triton.cdiv(bs * num_draft_tokens, 128),)
|
||||
](
|
||||
seq_lens,
|
||||
prefix,
|
||||
extend,
|
||||
post,
|
||||
bs,
|
||||
seq_lens.stride(0),
|
||||
num_draft_tokens,
|
||||
0,
|
||||
BLOCK=128,
|
||||
positions=positions,
|
||||
start_locs=starts,
|
||||
mrope=mrope,
|
||||
MAKE_POSITIONS=True,
|
||||
MAKE_MROPE=with_mrope,
|
||||
)
|
||||
return prefix, extend, post, (positions, starts, mrope)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["BS"])
|
||||
def _build_chain_tree_kernel(
|
||||
bonus,
|
||||
draft,
|
||||
seq,
|
||||
mask,
|
||||
positions,
|
||||
retrieve,
|
||||
tokens,
|
||||
BS,
|
||||
WIDTH: tl.constexpr,
|
||||
BONUS_STRIDE: tl.constexpr,
|
||||
DRAFT_ROW: tl.constexpr,
|
||||
DRAFT_COL: tl.constexpr,
|
||||
SEQ_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
req_indices,
|
||||
req_to_token,
|
||||
out_cache_loc,
|
||||
mrope,
|
||||
REQ_STRIDE: tl.constexpr,
|
||||
TABLE_ROW: tl.constexpr,
|
||||
TABLE_COL: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
WRITE_MROPE: tl.constexpr,
|
||||
):
|
||||
offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
|
||||
row = offset // WIDTH
|
||||
col = offset % WIDTH
|
||||
valid = offset < BS * WIDTH
|
||||
first = tl.load(bonus + row * BONUS_STRIDE, valid & (col == 0), other=0)
|
||||
rest = tl.load(
|
||||
draft + row * DRAFT_ROW + (col - 1) * DRAFT_COL, valid & (col > 0), other=0
|
||||
)
|
||||
tl.store(tokens + offset, tl.where(col == 0, first, rest), valid)
|
||||
length = tl.load(seq + row * SEQ_STRIDE, valid, other=0).to(tl.int64)
|
||||
tl.store(positions + offset, length + col, valid)
|
||||
if WRITE_CACHE:
|
||||
req = tl.load(req_indices + row * REQ_STRIDE, valid, other=0).to(tl.int64)
|
||||
loc = tl.load(
|
||||
req_to_token + req * TABLE_ROW + (length + col) * TABLE_COL, valid, other=0
|
||||
)
|
||||
tl.store(out_cache_loc + offset, loc, valid)
|
||||
if WRITE_MROPE:
|
||||
for axis in tl.static_range(3):
|
||||
tl.store(mrope + axis * BS * WIDTH + offset, length + col, valid)
|
||||
tl.store(retrieve + offset, offset, valid)
|
||||
tl.store(
|
||||
retrieve + BS * WIDTH + offset, tl.where(col + 1 < WIDTH, col + 1, -1), valid
|
||||
)
|
||||
tl.store(retrieve + 2 * BS * WIDTH + offset, -1, valid)
|
||||
mask_col = offset % WIDTH
|
||||
mask_row = (offset // WIDTH) % WIDTH
|
||||
tl.store(mask + offset, mask_col <= mask_row, offset < BS * WIDTH * WIDTH)
|
||||
|
||||
|
||||
def build_chain_tree(
|
||||
bonus_tokens,
|
||||
draft_tokens,
|
||||
seq_lens,
|
||||
tree_mask_buf=None,
|
||||
req_pool_indices=None,
|
||||
req_to_token=None,
|
||||
with_mrope=False,
|
||||
):
|
||||
bs, depth = draft_tokens.shape
|
||||
width = depth + 1
|
||||
assert bonus_tokens.numel() == seq_lens.numel() == bs
|
||||
mask = tree_mask_buf
|
||||
if mask is None:
|
||||
mask = torch.empty(
|
||||
(bs * width * width,), dtype=torch.bool, device=seq_lens.device
|
||||
)
|
||||
assert mask.is_contiguous() and mask.numel() >= bs * width * width
|
||||
positions = torch.empty((bs * width,), dtype=torch.int64, device=seq_lens.device)
|
||||
retrieve = torch.empty((3, bs, width), dtype=torch.int64, device=seq_lens.device)
|
||||
tokens = torch.empty(
|
||||
(bs * width,),
|
||||
dtype=torch.promote_types(bonus_tokens.dtype, draft_tokens.dtype),
|
||||
device=seq_lens.device,
|
||||
)
|
||||
out_cache_loc = torch.empty_like(positions) if req_to_token is not None else None
|
||||
mrope = (
|
||||
torch.empty((3, bs * width), dtype=torch.int64, device=seq_lens.device)
|
||||
if with_mrope
|
||||
else None
|
||||
)
|
||||
if req_to_token is not None:
|
||||
assert req_pool_indices is not None and req_pool_indices.numel() == bs
|
||||
if bs:
|
||||
_build_chain_tree_kernel[(triton.cdiv(bs * width * width, 128),)](
|
||||
bonus_tokens,
|
||||
draft_tokens,
|
||||
seq_lens,
|
||||
mask,
|
||||
positions,
|
||||
retrieve,
|
||||
tokens,
|
||||
bs,
|
||||
width,
|
||||
bonus_tokens.stride(0),
|
||||
draft_tokens.stride(0),
|
||||
draft_tokens.stride(1),
|
||||
seq_lens.stride(0),
|
||||
BLOCK=128,
|
||||
req_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
out_cache_loc=out_cache_loc,
|
||||
mrope=mrope,
|
||||
REQ_STRIDE=(
|
||||
req_pool_indices.stride(0) if req_pool_indices is not None else 0
|
||||
),
|
||||
TABLE_ROW=req_to_token.stride(0) if req_to_token is not None else 0,
|
||||
TABLE_COL=req_to_token.stride(1) if req_to_token is not None else 0,
|
||||
WRITE_CACHE=req_to_token is not None,
|
||||
WRITE_MROPE=with_mrope,
|
||||
)
|
||||
return (
|
||||
mask,
|
||||
positions,
|
||||
retrieve[0],
|
||||
retrieve[1],
|
||||
retrieve[2],
|
||||
tokens,
|
||||
out_cache_loc,
|
||||
mrope,
|
||||
)
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.attention.qsa.sparse_attn import (
|
||||
sparse_gqa_fwd_interface_triton_ck,
|
||||
sparse_gqa_packed_decode_triton,
|
||||
)
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
ContextParallelStrategyKind,
|
||||
cp_materialize_global_token_order,
|
||||
@@ -50,6 +51,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -252,6 +254,7 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
self._trtllm_counters = {}
|
||||
self._graph_extend_lens = None
|
||||
self._graph_extend_lens_pin = None
|
||||
self._verify_mask = None
|
||||
|
||||
@staticmethod
|
||||
def _is_speculative_paged_mode(forward_mode) -> bool:
|
||||
@@ -795,11 +798,24 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
num_padding=num_padding if num_padding is not None else 0,
|
||||
)
|
||||
|
||||
@property
|
||||
def verify_mask(self) -> Optional[VerifyMask]:
|
||||
return self._verify_mask
|
||||
|
||||
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int) -> None:
|
||||
if self.device is None:
|
||||
raise RuntimeError(
|
||||
"QSA backend requires a ModelRunner to initialize CUDA graph state"
|
||||
)
|
||||
self._verify_mask = maybe_create_verify_mask(
|
||||
is_draft_runner=self.runner is not None and self.runner.is_draft_worker,
|
||||
skip_prefill=False,
|
||||
max_bs=max_bs,
|
||||
max_context_len=self.max_context_len,
|
||||
num_draft_tokens=get_spec().speculative_num_draft_tokens,
|
||||
device=self.device,
|
||||
is_read=False,
|
||||
)
|
||||
self._cuda_graph_max_tokens = max_num_tokens
|
||||
max_blocks = math.ceil(self.max_context_len / self.compress_ratio)
|
||||
max_pages = max(
|
||||
|
||||
@@ -122,6 +122,9 @@ class GenerationBatchResult:
|
||||
# Every stage holds the KV for its own layers, so every stage has to compact
|
||||
# that path into its committed prefix; only the last stage can compute it.
|
||||
accept_index: Optional[torch.Tensor] = None
|
||||
prepared_draft_extend_inputs: Optional[
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
] = None
|
||||
|
||||
# Refs the worker wants scheduler to keep alive for the same 2-iter window
|
||||
# as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec
|
||||
|
||||
@@ -934,6 +934,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
*,
|
||||
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
|
||||
return_hidden_states_before_norm: bool,
|
||||
extend_position_info=None,
|
||||
spec_mrope_positions: Optional[torch.Tensor] = None,
|
||||
):
|
||||
# init_new must not mutate the input ScheduleBatch; per-forward
|
||||
# overrides go through explicit keyword arguments.
|
||||
@@ -1157,12 +1159,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
ret.extend_seq_lens = extend_seq_lens
|
||||
ret.extend_prefix_lens = extend_prefix_lens
|
||||
ret.extend_num_tokens = batch.extend_num_tokens
|
||||
positions, ret.extend_start_loc = compute_position(
|
||||
model_runner.prefill_attention_backend_str,
|
||||
ret.extend_prefix_lens,
|
||||
ret.extend_seq_lens,
|
||||
ret.extend_num_tokens,
|
||||
)
|
||||
if extend_position_info is None:
|
||||
positions, ret.extend_start_loc = compute_position(
|
||||
model_runner.prefill_attention_backend_str,
|
||||
ret.extend_prefix_lens,
|
||||
ret.extend_seq_lens,
|
||||
ret.extend_num_tokens,
|
||||
)
|
||||
else:
|
||||
positions, ret.extend_start_loc, _ = extend_position_info
|
||||
if ret.positions is None:
|
||||
ret.positions = positions
|
||||
ret.extend_logprob_start_lens_cpu = extend_logprob_start_lens
|
||||
@@ -1171,7 +1176,13 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
ret._init_ngram_embedding_info(batch, device)
|
||||
|
||||
if model_runner.model_config.model_is_mrope:
|
||||
if (
|
||||
if spec_mrope_positions is not None:
|
||||
ret.mrope_positions = spec_mrope_positions
|
||||
elif (
|
||||
extend_position_info is not None and extend_position_info[2] is not None
|
||||
):
|
||||
ret.mrope_positions = extend_position_info[2]
|
||||
elif (
|
||||
ret.spec_info is not None
|
||||
and getattr(ret.spec_info, "positions", None) is not None
|
||||
):
|
||||
@@ -1400,9 +1411,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
seq_positions = seq_positions.view(batch_size, -1)
|
||||
# Split text-only and mixed batches here because SpecV2 text-only batches can avoid an extra D2H.
|
||||
if all(mm_input is None for mm_input in mm_inputs):
|
||||
mrope_delta_tensor = torch.zeros(
|
||||
(batch_size, 1), dtype=torch.int64, device=device
|
||||
self.mrope_positions = (
|
||||
seq_positions.to(torch.int64).flatten().unsqueeze(0).repeat(3, 1)
|
||||
)
|
||||
return
|
||||
else:
|
||||
mrope_deltas = [
|
||||
(
|
||||
|
||||
@@ -624,7 +624,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
forward_batch.req_pool_indices,
|
||||
]
|
||||
if self.model_runner.model_config.model_is_mrope:
|
||||
buffers.mrope_positions.zero_()
|
||||
if bs != raw_bs or forward_batch.mrope_positions is None:
|
||||
buffers.mrope_positions.zero_()
|
||||
if forward_batch.mrope_positions is not None:
|
||||
copy_dsts.append(buffers.mrope_positions[:, :raw_num_token])
|
||||
copy_srcs.append(forward_batch.mrope_positions)
|
||||
|
||||
@@ -552,7 +552,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
forward_batch.req_pool_indices,
|
||||
]
|
||||
if self.model_runner.model_config.model_is_mrope:
|
||||
buffers.mrope_positions.zero_()
|
||||
if (
|
||||
bs * self.captured_req_width != num_tokens
|
||||
or forward_batch.mrope_positions is None
|
||||
):
|
||||
buffers.mrope_positions.zero_()
|
||||
if forward_batch.mrope_positions is not None:
|
||||
copy_dsts.append(buffers.mrope_positions[:, :num_tokens])
|
||||
copy_srcs.append(forward_batch.mrope_positions)
|
||||
|
||||
@@ -32,6 +32,8 @@ class EagleVerifyInput(SpecInput):
|
||||
# Stacked per-step draft proposal distribution q, shape (bs, num_steps,
|
||||
# vocab); only set under rejection sampling. Consumed by the verify kernel.
|
||||
draft_probs: torch.Tensor = None
|
||||
prepared_out_cache_loc: Optional[torch.Tensor] = None
|
||||
prepared_mrope_positions: Optional[torch.Tensor] = None
|
||||
|
||||
# Shape info for padding
|
||||
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
|
||||
|
||||
@@ -566,14 +566,17 @@ def eagle_prepare_for_verify(
|
||||
# Uniform variant: end offsets (= start + draft_token_num) are computed
|
||||
# inside the kernel, keeping the eager `seq_lens + N` add off the host
|
||||
# critical path (bs=1 MTP inter-phase seam).
|
||||
batch.out_cache_loc = assign_extend_cache_locs_uniform_func(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=req_to_token_pool.req_to_token,
|
||||
start_offset=batch.seq_lens,
|
||||
batch_size=bs,
|
||||
draft_token_num=verify_input.draft_token_num,
|
||||
device=device,
|
||||
)
|
||||
if verify_input.prepared_out_cache_loc is not None:
|
||||
batch.out_cache_loc = verify_input.prepared_out_cache_loc
|
||||
else:
|
||||
batch.out_cache_loc = assign_extend_cache_locs_uniform_func(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=req_to_token_pool.req_to_token,
|
||||
start_offset=batch.seq_lens,
|
||||
batch_size=bs,
|
||||
draft_token_num=verify_input.draft_token_num,
|
||||
device=device,
|
||||
)
|
||||
|
||||
batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle(
|
||||
batch, verify_input.draft_token_num
|
||||
@@ -601,6 +604,7 @@ def eagle_prepare_for_verify(
|
||||
target_worker.model_runner,
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=verify_input.prepared_mrope_positions,
|
||||
)
|
||||
|
||||
# Run attention backend plan and cuda graph preparation
|
||||
|
||||
@@ -7,7 +7,13 @@ import torch
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_draft_cache_locs_contiguous,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.eagle import fill_bonus_tokens_func
|
||||
from sglang.kernels.ops.speculative.eagle import (
|
||||
build_chain_tree,
|
||||
fill_bonus_tokens_func,
|
||||
prepare_draft_extend_layout,
|
||||
prepare_draft_extend_lengths,
|
||||
prepare_verify_commit_outputs,
|
||||
)
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -16,6 +22,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
ForwardMode,
|
||||
PPProxyTensors,
|
||||
)
|
||||
from sglang.srt.runtime_context import mamba_track_grid
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput
|
||||
from sglang.srt.speculative.eagle_utils import (
|
||||
TreeMaskMode,
|
||||
@@ -155,7 +162,22 @@ def prepare_for_draft_extend(
|
||||
)
|
||||
# init_new requires both list or both Tensor;
|
||||
# gpu_only emits device tensors to skip H2D.
|
||||
if gpu_only:
|
||||
fused_lengths = gpu_only and batch.seq_lens.is_cuda
|
||||
extend_position_info = None
|
||||
if fused_lengths and not widen and draft_extend_input.positions is None:
|
||||
batch.prefix_lens, batch.extend_lens, post_seq_lens, extend_position_info = (
|
||||
prepare_draft_extend_layout(
|
||||
batch.seq_lens,
|
||||
num_draft_tokens,
|
||||
draft_model_runner.model_config.model_is_mrope
|
||||
and all(x is None for x in batch.multimodal_inputs),
|
||||
)
|
||||
)
|
||||
elif fused_lengths:
|
||||
batch.prefix_lens, batch.extend_lens, post_seq_lens = (
|
||||
prepare_draft_extend_lengths(batch.seq_lens, num_draft_tokens, front_offset)
|
||||
)
|
||||
elif gpu_only:
|
||||
batch.prefix_lens = (batch.seq_lens - front_offset).clamp(min=0).to(torch.int32)
|
||||
batch.extend_lens = torch.full(
|
||||
(bs,), num_window_tokens, dtype=torch.int32, device=batch.seq_lens.device
|
||||
@@ -181,10 +203,13 @@ def prepare_for_draft_extend(
|
||||
draft_model_runner,
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=return_hidden_states_before_norm,
|
||||
extend_position_info=extend_position_info,
|
||||
)
|
||||
# Forward sees post-write length (draft extend writes num_draft_tokens
|
||||
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
|
||||
forward_batch.seq_lens = forward_batch.seq_lens + num_draft_tokens
|
||||
forward_batch.seq_lens = (
|
||||
post_seq_lens if fused_lengths else forward_batch.seq_lens + num_draft_tokens
|
||||
)
|
||||
if not gpu_only:
|
||||
forward_batch.seq_lens_cpu = forward_batch.seq_lens_cpu + num_draft_tokens
|
||||
forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum())
|
||||
@@ -220,6 +245,8 @@ def prepare_for_draft(
|
||||
num_steps: int,
|
||||
):
|
||||
|
||||
prepared_positions = None
|
||||
prepared_mrope_positions = None
|
||||
if not batch.forward_mode.is_idle():
|
||||
bs = len(batch.seq_lens)
|
||||
|
||||
@@ -242,6 +269,14 @@ def prepare_for_draft(
|
||||
num_steps,
|
||||
)
|
||||
else:
|
||||
if batch.seq_lens.is_cuda and topk == 1:
|
||||
prepared_positions = torch.empty_like(batch.seq_lens)
|
||||
if draft_model_runner.model_config.model_is_mrope and all(
|
||||
x is None for x in batch.multimodal_inputs
|
||||
):
|
||||
prepared_mrope_positions = torch.empty(
|
||||
(3, bs), dtype=torch.int64, device=batch.device
|
||||
)
|
||||
# FIXME(lsyin): align with the default code path
|
||||
assign_draft_cache_locs_contiguous[(bs,)](
|
||||
batch.req_pool_indices,
|
||||
@@ -251,6 +286,11 @@ def prepare_for_draft(
|
||||
req_to_token_pool.req_to_token.shape[1],
|
||||
topk,
|
||||
num_steps,
|
||||
positions=prepared_positions,
|
||||
mrope=prepared_mrope_positions,
|
||||
BS=bs,
|
||||
WRITE_POSITIONS=prepared_positions is not None,
|
||||
WRITE_MROPE=prepared_mrope_positions is not None,
|
||||
)
|
||||
else:
|
||||
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
|
||||
@@ -301,12 +341,17 @@ def prepare_for_draft(
|
||||
if draft_model_runner.spec_algorithm.is_standalone()
|
||||
else CaptureHiddenMode.LAST
|
||||
)
|
||||
draft_input.positions = batch.seq_lens.repeat_interleave(topk, dim=0)
|
||||
draft_input.positions = (
|
||||
prepared_positions
|
||||
if prepared_positions is not None
|
||||
else batch.seq_lens.repeat_interleave(topk, dim=0)
|
||||
)
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch,
|
||||
draft_model_runner,
|
||||
capture_hidden_mode=capture_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
spec_mrope_positions=prepared_mrope_positions,
|
||||
)
|
||||
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
|
||||
forward_batch
|
||||
@@ -364,6 +409,42 @@ def build_eagle_verify_input(
|
||||
else:
|
||||
seq_lens_sum = bs * target_attn_backend.max_context_len
|
||||
|
||||
prepared_out_cache_loc = None
|
||||
prepared_mrope_positions = None
|
||||
if (
|
||||
batch.seq_lens.is_cuda
|
||||
and topk == 1
|
||||
and num_draft_tokens == num_steps + 1
|
||||
and draft_tokens.shape == (bs, num_steps)
|
||||
and mask_mode == TreeMaskMode.QLEN_ONLY
|
||||
):
|
||||
tree_outputs = build_chain_tree(
|
||||
draft_input.bonus_tokens,
|
||||
draft_tokens,
|
||||
batch.seq_lens,
|
||||
tree_mask_buf,
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=batch.req_to_token_pool.req_to_token,
|
||||
with_mrope=target_worker.model_runner.model_config.model_is_mrope
|
||||
and all(x is None for x in batch.multimodal_inputs),
|
||||
)
|
||||
prepared_out_cache_loc, prepared_mrope_positions = tree_outputs[6:]
|
||||
tree_outputs = tree_outputs[:6]
|
||||
else:
|
||||
tree_outputs = build_tree_kernel_efficient(
|
||||
draft_input.bonus_tokens,
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
draft_tokens,
|
||||
batch.seq_lens,
|
||||
seq_lens_sum,
|
||||
topk,
|
||||
num_steps,
|
||||
num_draft_tokens,
|
||||
mask_mode,
|
||||
tree_mask_buf,
|
||||
fill_prefix_mask=fill_mask,
|
||||
)
|
||||
(
|
||||
tree_mask,
|
||||
position,
|
||||
@@ -371,20 +452,7 @@ def build_eagle_verify_input(
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
draft_tokens,
|
||||
) = build_tree_kernel_efficient(
|
||||
draft_input.bonus_tokens,
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
draft_tokens,
|
||||
batch.seq_lens,
|
||||
seq_lens_sum,
|
||||
topk,
|
||||
num_steps,
|
||||
num_draft_tokens,
|
||||
mask_mode,
|
||||
tree_mask_buf,
|
||||
fill_prefix_mask=fill_mask,
|
||||
)
|
||||
) = tree_outputs
|
||||
|
||||
return EagleVerifyInput(
|
||||
draft_token=draft_tokens,
|
||||
@@ -401,6 +469,8 @@ def build_eagle_verify_input(
|
||||
seq_lens_sum=None,
|
||||
seq_lens_cpu=None,
|
||||
draft_probs=draft_probs,
|
||||
prepared_out_cache_loc=prepared_out_cache_loc,
|
||||
prepared_mrope_positions=prepared_mrope_positions,
|
||||
)
|
||||
|
||||
|
||||
@@ -599,7 +669,30 @@ def run_eagle_verify(
|
||||
grammar_mask,
|
||||
uno_target_max_top_k=uno_target_max_top_k,
|
||||
)
|
||||
new_seq_lens = batch.seq_lens + accept_lens
|
||||
fused_commit_outputs = predict.is_cuda and not batch.forward_mode.is_idle()
|
||||
prepared_commit_steps = prepared_draft_inputs = None
|
||||
if fused_commit_outputs and topk == 1:
|
||||
track_interval = (
|
||||
mamba_track_grid(batch.tree_cache.page_size)
|
||||
if batch.mamba_track_indices is not None
|
||||
else 0
|
||||
)
|
||||
new_seq_lens, bonus_tokens, prepared_commit_steps, prepared_draft_inputs = (
|
||||
prepare_verify_commit_outputs(
|
||||
predict,
|
||||
accept_index,
|
||||
accept_lens,
|
||||
batch.seq_lens,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
mamba_track_interval=track_interval,
|
||||
)
|
||||
)
|
||||
elif fused_commit_outputs:
|
||||
new_seq_lens, bonus_tokens = prepare_verify_commit_outputs(
|
||||
predict, accept_index, accept_lens, batch.seq_lens
|
||||
)
|
||||
else:
|
||||
new_seq_lens = batch.seq_lens + accept_lens
|
||||
clear_unaccepted_c128 = getattr(
|
||||
token_to_kv_pool_allocator.get_kvcache(),
|
||||
"clear_unaccepted_c128_draft_states",
|
||||
@@ -620,9 +713,10 @@ def run_eagle_verify(
|
||||
accept_lens,
|
||||
accept_index,
|
||||
num_draft_tokens,
|
||||
prepared_step_indices=prepared_commit_steps,
|
||||
)
|
||||
|
||||
if not batch.forward_mode.is_idle():
|
||||
if not fused_commit_outputs and not batch.forward_mode.is_idle():
|
||||
accept_tokens = predict[accept_index]
|
||||
bonus_tokens = torch.empty_like(accept_lens, dtype=torch.int32)
|
||||
# stride = accept_tokens per-req width = accept_index.shape[1]
|
||||
@@ -634,7 +728,7 @@ def run_eagle_verify(
|
||||
accept_index.shape[1],
|
||||
bs,
|
||||
)
|
||||
else:
|
||||
elif batch.forward_mode.is_idle():
|
||||
bonus_tokens = torch.empty((0,), device=device, dtype=torch.int32)
|
||||
|
||||
if batch.return_logprob and not batch.forward_mode.is_idle():
|
||||
@@ -672,4 +766,5 @@ def run_eagle_verify(
|
||||
routed_experts_output=forward_batch_output.routed_experts_output,
|
||||
indexer_topk_output=forward_batch_output.indexer_topk_output,
|
||||
extra_keep_alive_refs=[verify_forward_batch],
|
||||
prepared_draft_extend_inputs=prepared_draft_inputs,
|
||||
)
|
||||
|
||||
@@ -1028,32 +1028,48 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
def _draft_extend_for_decode(
|
||||
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
|
||||
):
|
||||
# Cast to int64 before entering plan stream to avoid cross-stream
|
||||
# synchronization issues with .to() inside the plan stream context.
|
||||
if batch_result.prepared_draft_extend_inputs is not None:
|
||||
num_correct_drafts, select_index, next_token_ids = (
|
||||
batch_result.prepared_draft_extend_inputs
|
||||
)
|
||||
elif batch_result.accept_lens.is_cuda:
|
||||
from sglang.kernels.ops.speculative.eagle import prepare_draft_extend_inputs
|
||||
|
||||
num_correct_drafts, select_index, next_token_ids = (
|
||||
prepare_draft_extend_inputs(
|
||||
batch_result.accept_lens,
|
||||
batch_result.next_token_ids,
|
||||
self.speculative_num_draft_tokens,
|
||||
)
|
||||
)
|
||||
else:
|
||||
num_correct_drafts = batch_result.accept_lens - 1
|
||||
select_index = (
|
||||
torch.arange(
|
||||
0,
|
||||
len(batch.seq_lens) * self.speculative_num_draft_tokens,
|
||||
self.speculative_num_draft_tokens,
|
||||
device=self.device,
|
||||
)
|
||||
+ batch_result.accept_lens
|
||||
- 1
|
||||
)
|
||||
|
||||
next_token_ids = batch_result.next_token_ids.to(torch.int64)
|
||||
|
||||
# Batch 2: Draft extend
|
||||
draft_extend_input = EagleDraftExtendInput(
|
||||
hidden_states=batch_result.logits_output.hidden_states,
|
||||
# accept_lens includes the bonus token; correct drafts exclude it.
|
||||
num_correct_drafts=batch_result.accept_lens - 1,
|
||||
num_correct_drafts=num_correct_drafts,
|
||||
num_accept_tokens=batch_result.accept_lens,
|
||||
# Draft-extend fills the whole tree width (num_draft_tokens) per req,
|
||||
# not num_steps + 1, so DP MLP-sync padding stays consistent for topk > 1.
|
||||
num_tokens_per_req=self.speculative_num_draft_tokens,
|
||||
num_tokens_for_logprob_per_req=self.speculative_num_draft_tokens,
|
||||
)
|
||||
select_index = (
|
||||
torch.arange(
|
||||
0,
|
||||
len(batch.seq_lens) * self.speculative_num_draft_tokens,
|
||||
self.speculative_num_draft_tokens,
|
||||
device=self.device,
|
||||
)
|
||||
+ batch_result.accept_lens
|
||||
- 1
|
||||
)
|
||||
|
||||
# Cast to int64 before entering plan stream to avoid cross-stream
|
||||
# synchronization issues with .to() inside the plan stream context.
|
||||
next_token_ids = batch_result.next_token_ids.to(torch.int64)
|
||||
|
||||
# Prepare for draft extend in a separate stream
|
||||
if self.plan_stream:
|
||||
self.plan_stream.wait_stream(
|
||||
|
||||
@@ -313,6 +313,8 @@ def record_stream_for_v2_verify(batch, verify_input, fwd_stream):
|
||||
"draft_token",
|
||||
"custom_mask",
|
||||
"positions",
|
||||
"prepared_out_cache_loc",
|
||||
"prepared_mrope_positions",
|
||||
"retrieve_index",
|
||||
"retrieve_next_token",
|
||||
"retrieve_next_sibling",
|
||||
@@ -932,6 +934,7 @@ def commit_mamba_states_after_verify(
|
||||
accept_lens: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
draft_token_num: int,
|
||||
prepared_step_indices: Optional[Tuple[torch.Tensor, Optional[torch.Tensor]]] = None,
|
||||
) -> None:
|
||||
"""Commit accepted per-step mamba states into the persistent caches.
|
||||
|
||||
@@ -979,12 +982,17 @@ def commit_mamba_states_after_verify(
|
||||
|
||||
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
|
||||
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
|
||||
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
if prepared_step_indices is None:
|
||||
last_correct_step_indices, mamba_steps_to_track = (
|
||||
_verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
)
|
||||
else:
|
||||
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
|
||||
commit_gdn_replayssm_fold_after_verify(
|
||||
spec_state=spec_state,
|
||||
state_batch_indices=state_batch_indices,
|
||||
@@ -1015,12 +1023,17 @@ def commit_mamba_states_after_verify(
|
||||
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
|
||||
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
|
||||
replay_indices = batch.req_pool_indices
|
||||
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
if prepared_step_indices is None:
|
||||
last_correct_step_indices, mamba_steps_to_track = (
|
||||
_verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
)
|
||||
else:
|
||||
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
|
||||
# Advance the per-request circular cursors by the accepted count (incl. the
|
||||
# bonus token). max_cache_len = ring length L = replayssm_d.shape[-2].
|
||||
commit_gdn_replayssm_spec(
|
||||
@@ -1137,12 +1150,17 @@ def commit_mamba_states_after_verify(
|
||||
|
||||
# `accept_lens` already includes the bonus token (drafts + 1 per req).
|
||||
if not batch.forward_mode.is_idle() and accept_index.numel() > 0:
|
||||
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
if prepared_step_indices is None:
|
||||
last_correct_step_indices, mamba_steps_to_track = (
|
||||
_verify_commit_step_indices(
|
||||
batch=batch,
|
||||
accept_index=accept_index,
|
||||
accept_lens=accept_lens,
|
||||
draft_token_num=draft_token_num,
|
||||
)
|
||||
)
|
||||
else:
|
||||
last_correct_step_indices, mamba_steps_to_track = prepared_step_indices
|
||||
|
||||
if hasattr(attn_backend, "update_mamba_state_after_mtp_verify"):
|
||||
attn_backend.update_mamba_state_after_mtp_verify(
|
||||
|
||||
Reference in New Issue
Block a user