diff --git a/python/sglang/kernels/ops/speculative/cache_locs.py b/python/sglang/kernels/ops/speculative/cache_locs.py index 98697a020e..abf8bbb612 100644 --- a/python/sglang/kernels/ops/speculative/cache_locs.py +++ b/python/sglang/kernels/ops/speculative/cache_locs.py @@ -35,7 +35,7 @@ def _get_oot_speculative_cache_locs_fn() -> Callable[..., torch.Tensor] | None: return current_platform.get_speculative_cache_locs_fn() -@triton.jit +@triton.jit(do_not_specialize=["BS"]) def assign_draft_cache_locs_contiguous( req_pool_indices, req_to_token, @@ -44,6 +44,11 @@ def assign_draft_cache_locs_contiguous( pool_len: tl.constexpr, topk: tl.constexpr, speculative_num_steps: tl.constexpr, + positions=None, + mrope=None, + BS=0, + WRITE_POSITIONS: tl.constexpr = False, + WRITE_MROPE: tl.constexpr = False, ): BLOCK_SIZE: tl.constexpr = 128 pid = tl.program_id(axis=0) @@ -61,6 +66,17 @@ def assign_draft_cache_locs_contiguous( data = tl.load(token_pool + kv_start + copy_offset, mask=mask) tl.store(out_cache_ptr + copy_offset, data, mask=mask) + if WRITE_POSITIONS: + topk_offset = tl.arange(0, BLOCK_SIZE) + tl.store(positions + pid * topk + topk_offset, kv_start, topk_offset < topk) + if WRITE_MROPE: + for axis in tl.static_range(3): + tl.store( + mrope + axis * BS * topk + pid * topk + topk_offset, + kv_start, + topk_offset < topk, + ) + @triton.jit def generate_draft_decode_kv_indices( diff --git a/python/sglang/kernels/ops/speculative/eagle.py b/python/sglang/kernels/ops/speculative/eagle.py index 10342899ce..35247535cf 100644 --- a/python/sglang/kernels/ops/speculative/eagle.py +++ b/python/sglang/kernels/ops/speculative/eagle.py @@ -89,3 +89,394 @@ def fill_accept_out_cache_loc_func( accept_out_cache_loc, next_power_of_2(size), ) + + +@triton.jit(do_not_specialize=["BS", "PREDICT_SIZE"]) +def _verify_commit_outputs_kernel( + predict, + accept_index, + accept_lens, + seq_lens, + new_seq_lens, + bonus, + BS, + PREDICT_SIZE, + PREDICT_STRIDE: tl.constexpr, + INDEX_STRIDE_0: tl.constexpr, + INDEX_STRIDE_1: tl.constexpr, + ACCEPT_STRIDE: tl.constexpr, + SEQ_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, + last_steps, + track_steps, + correct, + select, + ids, + DRAFT_WIDTH: tl.constexpr, + TRACK_INTERVAL: tl.constexpr, +): + row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64) + valid = row < BS + accepted = tl.load(accept_lens + row * ACCEPT_STRIDE, valid, other=1).to(tl.int64) + length = tl.load(seq_lens + row * SEQ_STRIDE, valid, other=0).to(tl.int64) + idx = tl.load( + accept_index + row * INDEX_STRIDE_0 + (accepted - 1) * INDEX_STRIDE_1, + valid, + other=0, + ).to(tl.int64) + if DRAFT_WIDTH > 0: + tl.store(last_steps + row, idx - row * DRAFT_WIDTH, valid) + tl.store(correct + row, accepted - 1, valid) + tl.store(select + row, row * DRAFT_WIDTH + accepted - 1, valid) + output_token = tl.load( + predict + row * PREDICT_STRIDE, row < PREDICT_SIZE, other=0 + ) + tl.store(ids + row, output_token.to(tl.int64), row < PREDICT_SIZE) + if TRACK_INTERVAL > 0: + post = length + accepted + crossing = length // TRACK_INTERVAL != post // TRACK_INTERVAL + tracking_position = tl.maximum( + post // TRACK_INTERVAL * TRACK_INTERVAL - length - 1, 0 + ) + track = ( + tl.load( + accept_index + + row * INDEX_STRIDE_0 + + tracking_position * INDEX_STRIDE_1, + valid & crossing, + other=0, + ).to(tl.int64) + - row * DRAFT_WIDTH + ) + tl.store(track_steps + row, tl.where(crossing, track, -1), valid) + idx = tl.where(idx < 0, idx + PREDICT_SIZE, idx) + token = tl.load(predict + idx * PREDICT_STRIDE, valid, other=0) + tl.store(new_seq_lens + row, length + accepted, valid) + tl.store(bonus + row, token, valid) + + +def prepare_verify_commit_outputs( + predict, + accept_index, + accept_lens, + seq_lens, + num_draft_tokens=0, + mamba_track_interval=0, +): + bs = accept_lens.numel() + new_seq_lens = torch.empty( + (bs,), + device=seq_lens.device, + dtype=torch.promote_types(seq_lens.dtype, accept_lens.dtype), + ) + bonus = torch.empty((bs,), device=predict.device, dtype=torch.int32) + last_steps = track_steps = correct = select = ids = None + if num_draft_tokens: + last_steps = torch.empty((bs,), device=seq_lens.device, dtype=torch.int64) + if mamba_track_interval: + track_steps = torch.empty_like(last_steps) + correct = torch.empty_like(accept_lens, memory_format=torch.contiguous_format) + select = torch.empty_like(last_steps) + ids = torch.empty((predict.numel(),), device=predict.device, dtype=torch.int64) + if bs: + size = max(bs, predict.numel()) if num_draft_tokens else bs + _verify_commit_outputs_kernel[(triton.cdiv(size, 128),)]( + predict, + accept_index, + accept_lens, + seq_lens, + new_seq_lens, + bonus, + bs, + predict.numel(), + predict.stride(0), + accept_index.stride(0), + accept_index.stride(1), + accept_lens.stride(0), + seq_lens.stride(0), + BLOCK=128, + last_steps=last_steps, + track_steps=track_steps, + correct=correct, + select=select, + ids=ids, + DRAFT_WIDTH=num_draft_tokens, + TRACK_INTERVAL=mamba_track_interval, + ) + if num_draft_tokens: + return new_seq_lens, bonus, (last_steps, track_steps), (correct, select, ids) + return new_seq_lens, bonus + + +@triton.jit(do_not_specialize=["BS", "TOKENS"]) +def _draft_extend_inputs_kernel( + accept_lens, + predict, + correct, + select, + ids, + BS, + TOKENS, + WIDTH: tl.constexpr, + ACCEPT_STRIDE: tl.constexpr, + PREDICT_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64) + accepted = tl.load(accept_lens + offset * ACCEPT_STRIDE, offset < BS, other=1).to( + tl.int64 + ) + tl.store(correct + offset, accepted - 1, offset < BS) + tl.store(select + offset, offset * WIDTH + accepted - 1, offset < BS) + tokens = tl.load(predict + offset * PREDICT_STRIDE, offset < TOKENS, other=0) + tl.store(ids + offset, tokens.to(tl.int64), offset < TOKENS) + + +def prepare_draft_extend_inputs(accept_lens, predict, num_draft_tokens): + bs, tokens = accept_lens.numel(), predict.numel() + correct = torch.empty((bs,), device=accept_lens.device, dtype=accept_lens.dtype) + select = torch.empty((bs,), device=accept_lens.device, dtype=torch.int64) + ids = torch.empty((tokens,), device=predict.device, dtype=torch.int64) + if max(bs, tokens): + _draft_extend_inputs_kernel[(triton.cdiv(max(bs, tokens), 128),)]( + accept_lens, + predict, + correct, + select, + ids, + bs, + tokens, + num_draft_tokens, + accept_lens.stride(0), + predict.stride(0), + BLOCK=128, + ) + return correct, select, ids + + +@triton.jit(do_not_specialize=["BS"]) +def _prepare_draft_extend_lengths_kernel( + seq_lens, + prefix_lens, + extend_lens, + post_lens, + BS, + STRIDE: tl.constexpr, + WIDTH: tl.constexpr, + FRONT: tl.constexpr, + BLOCK: tl.constexpr, + positions=None, + start_locs=None, + mrope=None, + MAKE_POSITIONS: tl.constexpr = False, + MAKE_MROPE: tl.constexpr = False, +): + row = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64) + base = tl.load(seq_lens + row * STRIDE, row < BS, other=0).to(tl.int64) + tl.store(prefix_lens + row, tl.maximum(base - FRONT, 0), row < BS) + tl.store(extend_lens + row, WIDTH + FRONT, row < BS) + tl.store(post_lens + row, base + WIDTH, row < BS) + if MAKE_POSITIONS: + tl.static_assert(FRONT == 0) + tl.store(start_locs + row, row * WIDTH, row < BS) + req = row // WIDTH + token_base = tl.load(seq_lens + req * STRIDE, row < BS * WIDTH, other=0).to( + tl.int64 + ) + prefix = tl.maximum(token_base - FRONT, 0).to(tl.int32) + position = prefix + (row % WIDTH).to(tl.int32) + tl.store(positions + row, position.to(tl.int64), row < BS * WIDTH) + if MAKE_MROPE: + for axis in tl.static_range(3): + tl.store( + mrope + axis * BS * WIDTH + row, + position.to(tl.int64), + row < BS * WIDTH, + ) + + +def prepare_draft_extend_lengths(seq_lens, num_draft_tokens, front_offset): + bs = seq_lens.numel() + prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device) + extend = torch.empty_like(prefix) + post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device) + if bs: + _prepare_draft_extend_lengths_kernel[(triton.cdiv(bs, 128),)]( + seq_lens, + prefix, + extend, + post, + bs, + seq_lens.stride(0), + num_draft_tokens, + front_offset, + BLOCK=128, + ) + return prefix, extend, post + + +def prepare_draft_extend_layout(seq_lens, num_draft_tokens, with_mrope): + bs = seq_lens.numel() + prefix = torch.empty((bs,), dtype=torch.int32, device=seq_lens.device) + extend = torch.empty_like(prefix) + post = torch.empty((bs,), dtype=seq_lens.dtype, device=seq_lens.device) + positions = torch.empty( + (bs * num_draft_tokens,), dtype=torch.int64, device=seq_lens.device + ) + starts = torch.empty_like(prefix) + mrope = ( + torch.empty( + (3, bs * num_draft_tokens), dtype=torch.int64, device=seq_lens.device + ) + if with_mrope + else None + ) + if bs: + _prepare_draft_extend_lengths_kernel[ + (triton.cdiv(bs * num_draft_tokens, 128),) + ]( + seq_lens, + prefix, + extend, + post, + bs, + seq_lens.stride(0), + num_draft_tokens, + 0, + BLOCK=128, + positions=positions, + start_locs=starts, + mrope=mrope, + MAKE_POSITIONS=True, + MAKE_MROPE=with_mrope, + ) + return prefix, extend, post, (positions, starts, mrope) + + +@triton.jit(do_not_specialize=["BS"]) +def _build_chain_tree_kernel( + bonus, + draft, + seq, + mask, + positions, + retrieve, + tokens, + BS, + WIDTH: tl.constexpr, + BONUS_STRIDE: tl.constexpr, + DRAFT_ROW: tl.constexpr, + DRAFT_COL: tl.constexpr, + SEQ_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, + req_indices, + req_to_token, + out_cache_loc, + mrope, + REQ_STRIDE: tl.constexpr, + TABLE_ROW: tl.constexpr, + TABLE_COL: tl.constexpr, + WRITE_CACHE: tl.constexpr, + WRITE_MROPE: tl.constexpr, +): + offset = (tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64) + row = offset // WIDTH + col = offset % WIDTH + valid = offset < BS * WIDTH + first = tl.load(bonus + row * BONUS_STRIDE, valid & (col == 0), other=0) + rest = tl.load( + draft + row * DRAFT_ROW + (col - 1) * DRAFT_COL, valid & (col > 0), other=0 + ) + tl.store(tokens + offset, tl.where(col == 0, first, rest), valid) + length = tl.load(seq + row * SEQ_STRIDE, valid, other=0).to(tl.int64) + tl.store(positions + offset, length + col, valid) + if WRITE_CACHE: + req = tl.load(req_indices + row * REQ_STRIDE, valid, other=0).to(tl.int64) + loc = tl.load( + req_to_token + req * TABLE_ROW + (length + col) * TABLE_COL, valid, other=0 + ) + tl.store(out_cache_loc + offset, loc, valid) + if WRITE_MROPE: + for axis in tl.static_range(3): + tl.store(mrope + axis * BS * WIDTH + offset, length + col, valid) + tl.store(retrieve + offset, offset, valid) + tl.store( + retrieve + BS * WIDTH + offset, tl.where(col + 1 < WIDTH, col + 1, -1), valid + ) + tl.store(retrieve + 2 * BS * WIDTH + offset, -1, valid) + mask_col = offset % WIDTH + mask_row = (offset // WIDTH) % WIDTH + tl.store(mask + offset, mask_col <= mask_row, offset < BS * WIDTH * WIDTH) + + +def build_chain_tree( + bonus_tokens, + draft_tokens, + seq_lens, + tree_mask_buf=None, + req_pool_indices=None, + req_to_token=None, + with_mrope=False, +): + bs, depth = draft_tokens.shape + width = depth + 1 + assert bonus_tokens.numel() == seq_lens.numel() == bs + mask = tree_mask_buf + if mask is None: + mask = torch.empty( + (bs * width * width,), dtype=torch.bool, device=seq_lens.device + ) + assert mask.is_contiguous() and mask.numel() >= bs * width * width + positions = torch.empty((bs * width,), dtype=torch.int64, device=seq_lens.device) + retrieve = torch.empty((3, bs, width), dtype=torch.int64, device=seq_lens.device) + tokens = torch.empty( + (bs * width,), + dtype=torch.promote_types(bonus_tokens.dtype, draft_tokens.dtype), + device=seq_lens.device, + ) + out_cache_loc = torch.empty_like(positions) if req_to_token is not None else None + mrope = ( + torch.empty((3, bs * width), dtype=torch.int64, device=seq_lens.device) + if with_mrope + else None + ) + if req_to_token is not None: + assert req_pool_indices is not None and req_pool_indices.numel() == bs + if bs: + _build_chain_tree_kernel[(triton.cdiv(bs * width * width, 128),)]( + bonus_tokens, + draft_tokens, + seq_lens, + mask, + positions, + retrieve, + tokens, + bs, + width, + bonus_tokens.stride(0), + draft_tokens.stride(0), + draft_tokens.stride(1), + seq_lens.stride(0), + BLOCK=128, + req_indices=req_pool_indices, + req_to_token=req_to_token, + out_cache_loc=out_cache_loc, + mrope=mrope, + REQ_STRIDE=( + req_pool_indices.stride(0) if req_pool_indices is not None else 0 + ), + TABLE_ROW=req_to_token.stride(0) if req_to_token is not None else 0, + TABLE_COL=req_to_token.stride(1) if req_to_token is not None else 0, + WRITE_CACHE=req_to_token is not None, + WRITE_MROPE=with_mrope, + ) + return ( + mask, + positions, + retrieve[0], + retrieve[1], + retrieve[2], + tokens, + out_cache_loc, + mrope, + ) diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 82de8d9f29..127fb65950 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -41,6 +41,7 @@ from sglang.srt.layers.attention.qsa.sparse_attn import ( sparse_gqa_fwd_interface_triton_ck, sparse_gqa_packed_decode_triton, ) +from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask from sglang.srt.layers.cp.utils import ( ContextParallelStrategyKind, cp_materialize_global_token_order, @@ -50,6 +51,7 @@ from sglang.srt.layers.cp.utils import ( from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_spec from sglang.srt.utils import is_hip logger = logging.getLogger(__name__) @@ -252,6 +254,7 @@ class QwenSparseAttnBackend(AttentionBackend): self._trtllm_counters = {} self._graph_extend_lens = None self._graph_extend_lens_pin = None + self._verify_mask = None @staticmethod def _is_speculative_paged_mode(forward_mode) -> bool: @@ -795,11 +798,24 @@ class QwenSparseAttnBackend(AttentionBackend): num_padding=num_padding if num_padding is not None else 0, ) + @property + def verify_mask(self) -> Optional[VerifyMask]: + return self._verify_mask + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int) -> None: if self.device is None: raise RuntimeError( "QSA backend requires a ModelRunner to initialize CUDA graph state" ) + self._verify_mask = maybe_create_verify_mask( + is_draft_runner=self.runner is not None and self.runner.is_draft_worker, + skip_prefill=False, + max_bs=max_bs, + max_context_len=self.max_context_len, + num_draft_tokens=get_spec().speculative_num_draft_tokens, + device=self.device, + is_read=False, + ) self._cuda_graph_max_tokens = max_num_tokens max_blocks = math.ceil(self.max_context_len / self.compress_ratio) max_pages = max( diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 5c405dff91..90c7ba9fe2 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -122,6 +122,9 @@ class GenerationBatchResult: # Every stage holds the KV for its own layers, so every stage has to compact # that path into its committed prefix; only the last stage can compute it. accept_index: Optional[torch.Tensor] = None + prepared_draft_extend_inputs: Optional[ + tuple[torch.Tensor, torch.Tensor, torch.Tensor] + ] = None # Refs the worker wants scheduler to keep alive for the same 2-iter window # as batch_record_buf. Used for cross-stream tensor lifetime (e.g. a spec diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 09351a2b80..4acfcc71c9 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -934,6 +934,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): *, capture_hidden_mode: Optional[CaptureHiddenMode] = None, return_hidden_states_before_norm: bool, + extend_position_info=None, + spec_mrope_positions: Optional[torch.Tensor] = None, ): # init_new must not mutate the input ScheduleBatch; per-forward # overrides go through explicit keyword arguments. @@ -1157,12 +1159,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ret.extend_seq_lens = extend_seq_lens ret.extend_prefix_lens = extend_prefix_lens ret.extend_num_tokens = batch.extend_num_tokens - positions, ret.extend_start_loc = compute_position( - model_runner.prefill_attention_backend_str, - ret.extend_prefix_lens, - ret.extend_seq_lens, - ret.extend_num_tokens, - ) + if extend_position_info is None: + positions, ret.extend_start_loc = compute_position( + model_runner.prefill_attention_backend_str, + ret.extend_prefix_lens, + ret.extend_seq_lens, + ret.extend_num_tokens, + ) + else: + positions, ret.extend_start_loc, _ = extend_position_info if ret.positions is None: ret.positions = positions ret.extend_logprob_start_lens_cpu = extend_logprob_start_lens @@ -1171,7 +1176,13 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ret._init_ngram_embedding_info(batch, device) if model_runner.model_config.model_is_mrope: - if ( + if spec_mrope_positions is not None: + ret.mrope_positions = spec_mrope_positions + elif ( + extend_position_info is not None and extend_position_info[2] is not None + ): + ret.mrope_positions = extend_position_info[2] + elif ( ret.spec_info is not None and getattr(ret.spec_info, "positions", None) is not None ): @@ -1400,9 +1411,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): seq_positions = seq_positions.view(batch_size, -1) # Split text-only and mixed batches here because SpecV2 text-only batches can avoid an extra D2H. if all(mm_input is None for mm_input in mm_inputs): - mrope_delta_tensor = torch.zeros( - (batch_size, 1), dtype=torch.int64, device=device + self.mrope_positions = ( + seq_positions.to(torch.int64).flatten().unsqueeze(0).repeat(3, 1) ) + return else: mrope_deltas = [ ( diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 820aff9ae8..1896b29750 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -624,7 +624,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): forward_batch.req_pool_indices, ] if self.model_runner.model_config.model_is_mrope: - buffers.mrope_positions.zero_() + if bs != raw_bs or forward_batch.mrope_positions is None: + buffers.mrope_positions.zero_() if forward_batch.mrope_positions is not None: copy_dsts.append(buffers.mrope_positions[:, :raw_num_token]) copy_srcs.append(forward_batch.mrope_positions) diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index c72dda1541..3dc941524b 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -552,7 +552,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): forward_batch.req_pool_indices, ] if self.model_runner.model_config.model_is_mrope: - buffers.mrope_positions.zero_() + if ( + bs * self.captured_req_width != num_tokens + or forward_batch.mrope_positions is None + ): + buffers.mrope_positions.zero_() if forward_batch.mrope_positions is not None: copy_dsts.append(buffers.mrope_positions[:, :num_tokens]) copy_srcs.append(forward_batch.mrope_positions) diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index f28260586c..1e2f5e5b4b 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -32,6 +32,8 @@ class EagleVerifyInput(SpecInput): # Stacked per-step draft proposal distribution q, shape (bs, num_steps, # vocab); only set under rejection sampling. Consumed by the verify kernel. draft_probs: torch.Tensor = None + prepared_out_cache_loc: Optional[torch.Tensor] = None + prepared_mrope_positions: Optional[torch.Tensor] = None # Shape info for padding num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num. diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 91ee6212b5..a312a8b403 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -566,14 +566,17 @@ def eagle_prepare_for_verify( # Uniform variant: end offsets (= start + draft_token_num) are computed # inside the kernel, keeping the eager `seq_lens + N` add off the host # critical path (bs=1 MTP inter-phase seam). - batch.out_cache_loc = assign_extend_cache_locs_uniform_func( - req_pool_indices=batch.req_pool_indices, - req_to_token=req_to_token_pool.req_to_token, - start_offset=batch.seq_lens, - batch_size=bs, - draft_token_num=verify_input.draft_token_num, - device=device, - ) + if verify_input.prepared_out_cache_loc is not None: + batch.out_cache_loc = verify_input.prepared_out_cache_loc + else: + batch.out_cache_loc = assign_extend_cache_locs_uniform_func( + req_pool_indices=batch.req_pool_indices, + req_to_token=req_to_token_pool.req_to_token, + start_offset=batch.seq_lens, + batch_size=bs, + draft_token_num=verify_input.draft_token_num, + device=device, + ) batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle( batch, verify_input.draft_token_num @@ -601,6 +604,7 @@ def eagle_prepare_for_verify( target_worker.model_runner, capture_hidden_mode=capture_mode, return_hidden_states_before_norm=False, + spec_mrope_positions=verify_input.prepared_mrope_positions, ) # Run attention backend plan and cuda graph preparation diff --git a/python/sglang/srt/speculative/eagle_worker_common.py b/python/sglang/srt/speculative/eagle_worker_common.py index 0c2c530860..7120128866 100644 --- a/python/sglang/srt/speculative/eagle_worker_common.py +++ b/python/sglang/srt/speculative/eagle_worker_common.py @@ -7,7 +7,13 @@ import torch from sglang.kernels.ops.speculative.cache_locs import ( assign_draft_cache_locs_contiguous, ) -from sglang.kernels.ops.speculative.eagle import fill_bonus_tokens_func +from sglang.kernels.ops.speculative.eagle import ( + build_chain_tree, + fill_bonus_tokens_func, + prepare_draft_extend_layout, + prepare_draft_extend_lengths, + prepare_verify_commit_outputs, +) from sglang.srt.layers.logprob_processor import compute_spec_logprobs from sglang.srt.managers.utils import GenerationBatchResult from sglang.srt.model_executor.forward_batch_info import ( @@ -16,6 +22,7 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardMode, PPProxyTensors, ) +from sglang.srt.runtime_context import mamba_track_grid from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput from sglang.srt.speculative.eagle_utils import ( TreeMaskMode, @@ -155,7 +162,22 @@ def prepare_for_draft_extend( ) # init_new requires both list or both Tensor; # gpu_only emits device tensors to skip H2D. - if gpu_only: + fused_lengths = gpu_only and batch.seq_lens.is_cuda + extend_position_info = None + if fused_lengths and not widen and draft_extend_input.positions is None: + batch.prefix_lens, batch.extend_lens, post_seq_lens, extend_position_info = ( + prepare_draft_extend_layout( + batch.seq_lens, + num_draft_tokens, + draft_model_runner.model_config.model_is_mrope + and all(x is None for x in batch.multimodal_inputs), + ) + ) + elif fused_lengths: + batch.prefix_lens, batch.extend_lens, post_seq_lens = ( + prepare_draft_extend_lengths(batch.seq_lens, num_draft_tokens, front_offset) + ) + elif gpu_only: batch.prefix_lens = (batch.seq_lens - front_offset).clamp(min=0).to(torch.int32) batch.extend_lens = torch.full( (bs,), num_window_tokens, dtype=torch.int32, device=batch.seq_lens.device @@ -181,10 +203,13 @@ def prepare_for_draft_extend( draft_model_runner, capture_hidden_mode=capture_mode, return_hidden_states_before_norm=return_hidden_states_before_norm, + extend_position_info=extend_position_info, ) # Forward sees post-write length (draft extend writes num_draft_tokens # slots); mutation stays on forward_batch to preserve SB.seq_lens. - forward_batch.seq_lens = forward_batch.seq_lens + num_draft_tokens + forward_batch.seq_lens = ( + post_seq_lens if fused_lengths else forward_batch.seq_lens + num_draft_tokens + ) if not gpu_only: forward_batch.seq_lens_cpu = forward_batch.seq_lens_cpu + num_draft_tokens forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum()) @@ -220,6 +245,8 @@ def prepare_for_draft( num_steps: int, ): + prepared_positions = None + prepared_mrope_positions = None if not batch.forward_mode.is_idle(): bs = len(batch.seq_lens) @@ -242,6 +269,14 @@ def prepare_for_draft( num_steps, ) else: + if batch.seq_lens.is_cuda and topk == 1: + prepared_positions = torch.empty_like(batch.seq_lens) + if draft_model_runner.model_config.model_is_mrope and all( + x is None for x in batch.multimodal_inputs + ): + prepared_mrope_positions = torch.empty( + (3, bs), dtype=torch.int64, device=batch.device + ) # FIXME(lsyin): align with the default code path assign_draft_cache_locs_contiguous[(bs,)]( batch.req_pool_indices, @@ -251,6 +286,11 @@ def prepare_for_draft( req_to_token_pool.req_to_token.shape[1], topk, num_steps, + positions=prepared_positions, + mrope=prepared_mrope_positions, + BS=bs, + WRITE_POSITIONS=prepared_positions is not None, + WRITE_MROPE=prepared_mrope_positions is not None, ) else: # page_size > 1 + topk > 1: per-branch page-aligned draft pages. @@ -301,12 +341,17 @@ def prepare_for_draft( if draft_model_runner.spec_algorithm.is_standalone() else CaptureHiddenMode.LAST ) - draft_input.positions = batch.seq_lens.repeat_interleave(topk, dim=0) + draft_input.positions = ( + prepared_positions + if prepared_positions is not None + else batch.seq_lens.repeat_interleave(topk, dim=0) + ) forward_batch = ForwardBatch.init_new( batch, draft_model_runner, capture_hidden_mode=capture_mode, return_hidden_states_before_norm=False, + spec_mrope_positions=prepared_mrope_positions, ) can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph( forward_batch @@ -364,6 +409,42 @@ def build_eagle_verify_input( else: seq_lens_sum = bs * target_attn_backend.max_context_len + prepared_out_cache_loc = None + prepared_mrope_positions = None + if ( + batch.seq_lens.is_cuda + and topk == 1 + and num_draft_tokens == num_steps + 1 + and draft_tokens.shape == (bs, num_steps) + and mask_mode == TreeMaskMode.QLEN_ONLY + ): + tree_outputs = build_chain_tree( + draft_input.bonus_tokens, + draft_tokens, + batch.seq_lens, + tree_mask_buf, + req_pool_indices=batch.req_pool_indices, + req_to_token=batch.req_to_token_pool.req_to_token, + with_mrope=target_worker.model_runner.model_config.model_is_mrope + and all(x is None for x in batch.multimodal_inputs), + ) + prepared_out_cache_loc, prepared_mrope_positions = tree_outputs[6:] + tree_outputs = tree_outputs[:6] + else: + tree_outputs = build_tree_kernel_efficient( + draft_input.bonus_tokens, + parent_list, + top_scores_index, + draft_tokens, + batch.seq_lens, + seq_lens_sum, + topk, + num_steps, + num_draft_tokens, + mask_mode, + tree_mask_buf, + fill_prefix_mask=fill_mask, + ) ( tree_mask, position, @@ -371,20 +452,7 @@ def build_eagle_verify_input( retrieve_next_token, retrieve_next_sibling, draft_tokens, - ) = build_tree_kernel_efficient( - draft_input.bonus_tokens, - parent_list, - top_scores_index, - draft_tokens, - batch.seq_lens, - seq_lens_sum, - topk, - num_steps, - num_draft_tokens, - mask_mode, - tree_mask_buf, - fill_prefix_mask=fill_mask, - ) + ) = tree_outputs return EagleVerifyInput( draft_token=draft_tokens, @@ -401,6 +469,8 @@ def build_eagle_verify_input( seq_lens_sum=None, seq_lens_cpu=None, draft_probs=draft_probs, + prepared_out_cache_loc=prepared_out_cache_loc, + prepared_mrope_positions=prepared_mrope_positions, ) @@ -599,7 +669,30 @@ def run_eagle_verify( grammar_mask, uno_target_max_top_k=uno_target_max_top_k, ) - new_seq_lens = batch.seq_lens + accept_lens + fused_commit_outputs = predict.is_cuda and not batch.forward_mode.is_idle() + prepared_commit_steps = prepared_draft_inputs = None + if fused_commit_outputs and topk == 1: + track_interval = ( + mamba_track_grid(batch.tree_cache.page_size) + if batch.mamba_track_indices is not None + else 0 + ) + new_seq_lens, bonus_tokens, prepared_commit_steps, prepared_draft_inputs = ( + prepare_verify_commit_outputs( + predict, + accept_index, + accept_lens, + batch.seq_lens, + num_draft_tokens=num_draft_tokens, + mamba_track_interval=track_interval, + ) + ) + elif fused_commit_outputs: + new_seq_lens, bonus_tokens = prepare_verify_commit_outputs( + predict, accept_index, accept_lens, batch.seq_lens + ) + else: + new_seq_lens = batch.seq_lens + accept_lens clear_unaccepted_c128 = getattr( token_to_kv_pool_allocator.get_kvcache(), "clear_unaccepted_c128_draft_states", @@ -620,9 +713,10 @@ def run_eagle_verify( accept_lens, accept_index, num_draft_tokens, + prepared_step_indices=prepared_commit_steps, ) - if not batch.forward_mode.is_idle(): + if not fused_commit_outputs and not batch.forward_mode.is_idle(): accept_tokens = predict[accept_index] bonus_tokens = torch.empty_like(accept_lens, dtype=torch.int32) # stride = accept_tokens per-req width = accept_index.shape[1] @@ -634,7 +728,7 @@ def run_eagle_verify( accept_index.shape[1], bs, ) - else: + elif batch.forward_mode.is_idle(): bonus_tokens = torch.empty((0,), device=device, dtype=torch.int32) if batch.return_logprob and not batch.forward_mode.is_idle(): @@ -672,4 +766,5 @@ def run_eagle_verify( routed_experts_output=forward_batch_output.routed_experts_output, indexer_topk_output=forward_batch_output.indexer_topk_output, extra_keep_alive_refs=[verify_forward_batch], + prepared_draft_extend_inputs=prepared_draft_inputs, ) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 530ada37e5..7afc056dd0 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -1028,32 +1028,48 @@ class EagleDraftWorker(EagleDraftWorkerBase): def _draft_extend_for_decode( self, batch: ScheduleBatch, batch_result: GenerationBatchResult ): + # Cast to int64 before entering plan stream to avoid cross-stream + # synchronization issues with .to() inside the plan stream context. + if batch_result.prepared_draft_extend_inputs is not None: + num_correct_drafts, select_index, next_token_ids = ( + batch_result.prepared_draft_extend_inputs + ) + elif batch_result.accept_lens.is_cuda: + from sglang.kernels.ops.speculative.eagle import prepare_draft_extend_inputs + + num_correct_drafts, select_index, next_token_ids = ( + prepare_draft_extend_inputs( + batch_result.accept_lens, + batch_result.next_token_ids, + self.speculative_num_draft_tokens, + ) + ) + else: + num_correct_drafts = batch_result.accept_lens - 1 + select_index = ( + torch.arange( + 0, + len(batch.seq_lens) * self.speculative_num_draft_tokens, + self.speculative_num_draft_tokens, + device=self.device, + ) + + batch_result.accept_lens + - 1 + ) + + next_token_ids = batch_result.next_token_ids.to(torch.int64) + # Batch 2: Draft extend draft_extend_input = EagleDraftExtendInput( hidden_states=batch_result.logits_output.hidden_states, # accept_lens includes the bonus token; correct drafts exclude it. - num_correct_drafts=batch_result.accept_lens - 1, + num_correct_drafts=num_correct_drafts, num_accept_tokens=batch_result.accept_lens, # Draft-extend fills the whole tree width (num_draft_tokens) per req, # not num_steps + 1, so DP MLP-sync padding stays consistent for topk > 1. num_tokens_per_req=self.speculative_num_draft_tokens, num_tokens_for_logprob_per_req=self.speculative_num_draft_tokens, ) - select_index = ( - torch.arange( - 0, - len(batch.seq_lens) * self.speculative_num_draft_tokens, - self.speculative_num_draft_tokens, - device=self.device, - ) - + batch_result.accept_lens - - 1 - ) - - # Cast to int64 before entering plan stream to avoid cross-stream - # synchronization issues with .to() inside the plan stream context. - next_token_ids = batch_result.next_token_ids.to(torch.int64) - # Prepare for draft extend in a separate stream if self.plan_stream: self.plan_stream.wait_stream( diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index d6ef5cb441..1c5390b00f 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -313,6 +313,8 @@ def record_stream_for_v2_verify(batch, verify_input, fwd_stream): "draft_token", "custom_mask", "positions", + "prepared_out_cache_loc", + "prepared_mrope_positions", "retrieve_index", "retrieve_next_token", "retrieve_next_sibling", @@ -932,6 +934,7 @@ def commit_mamba_states_after_verify( accept_lens: torch.Tensor, accept_index: torch.Tensor, draft_token_num: int, + prepared_step_indices: Optional[Tuple[torch.Tensor, Optional[torch.Tensor]]] = None, ) -> None: """Commit accepted per-step mamba states into the persistent caches. @@ -979,12 +982,17 @@ def commit_mamba_states_after_verify( spec_state = req_pool.get_speculative_mamba2_params_all_layers() state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices) - last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices( - batch=batch, - accept_index=accept_index, - accept_lens=accept_lens, - draft_token_num=draft_token_num, - ) + if prepared_step_indices is None: + last_correct_step_indices, mamba_steps_to_track = ( + _verify_commit_step_indices( + batch=batch, + accept_index=accept_index, + accept_lens=accept_lens, + draft_token_num=draft_token_num, + ) + ) + else: + last_correct_step_indices, mamba_steps_to_track = prepared_step_indices commit_gdn_replayssm_fold_after_verify( spec_state=spec_state, state_batch_indices=state_batch_indices, @@ -1015,12 +1023,17 @@ def commit_mamba_states_after_verify( spec_state = req_pool.get_speculative_mamba2_params_all_layers() state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices) replay_indices = batch.req_pool_indices - last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices( - batch=batch, - accept_index=accept_index, - accept_lens=accept_lens, - draft_token_num=draft_token_num, - ) + if prepared_step_indices is None: + last_correct_step_indices, mamba_steps_to_track = ( + _verify_commit_step_indices( + batch=batch, + accept_index=accept_index, + accept_lens=accept_lens, + draft_token_num=draft_token_num, + ) + ) + else: + last_correct_step_indices, mamba_steps_to_track = prepared_step_indices # Advance the per-request circular cursors by the accepted count (incl. the # bonus token). max_cache_len = ring length L = replayssm_d.shape[-2]. commit_gdn_replayssm_spec( @@ -1137,12 +1150,17 @@ def commit_mamba_states_after_verify( # `accept_lens` already includes the bonus token (drafts + 1 per req). if not batch.forward_mode.is_idle() and accept_index.numel() > 0: - last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices( - batch=batch, - accept_index=accept_index, - accept_lens=accept_lens, - draft_token_num=draft_token_num, - ) + if prepared_step_indices is None: + last_correct_step_indices, mamba_steps_to_track = ( + _verify_commit_step_indices( + batch=batch, + accept_index=accept_index, + accept_lens=accept_lens, + draft_token_num=draft_token_num, + ) + ) + else: + last_correct_step_indices, mamba_steps_to_track = prepared_step_indices if hasattr(attn_backend, "update_mamba_state_after_mtp_verify"): attn_backend.update_mamba_state_after_mtp_verify( diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py b/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py index db8dca758d..6ae2cf3947 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py @@ -358,7 +358,7 @@ def test_fused_pool_eligibility(dtype, quantized, wrapped): "mode_name", ["DECODE", "TARGET_VERIFY", "DRAFT_EXTEND_V2", "EXTEND"] ) @pytest.mark.parametrize("ratio,budget", [(4, 2048), (2, 1024), (4, 8192)]) -def test_defer_expansion_metadata(mode_name, ratio, budget): +def test_defer_expansion_metadata(mode_name, ratio, budget, monkeypatch): from types import SimpleNamespace from sglang.srt.layers.attention.qwen_sparse_attn_backend import ( @@ -390,12 +390,17 @@ def test_defer_expansion_metadata(mode_name, ratio, budget): runner = SimpleNamespace( token_to_kv_pool=pool, device="cuda", + is_draft_worker=False, req_to_token_pool=SimpleNamespace(req_to_token=table), model_config=SimpleNamespace(hf_config=config, context_len=64), ) backend = QwenSparseAttnBackend(runner) mode = getattr(ForwardMode, mode_name) width = 1 if mode.is_decode() else 2 + monkeypatch.setattr( + "sglang.srt.layers.attention.qwen_sparse_attn_backend.get_spec", + lambda: SimpleNamespace(speculative_num_draft_tokens=width), + ) lengths = torch.tensor([8], device="cuda", dtype=torch.int32) batch = SimpleNamespace( forward_mode=mode, @@ -421,6 +426,9 @@ def test_defer_expansion_metadata(mode_name, ratio, budget): assert metadata.indexer_metadata.defer_block_expansion == expected if not mode.is_extend_without_speculative(): backend.init_cuda_graph_state(1, width) + assert backend.verify_mask is not None + assert not backend.verify_mask.is_read + assert backend.verify_mask.buffer.numel() == width * width backend.token_to_kv_pool = None backend._fused_kv_pool_eligible = False backend._capture_cuda_graph_metadata( diff --git a/test/registered/kernels/ops/speculative/test_next_verify_prep.py b/test/registered/kernels/ops/speculative/test_next_verify_prep.py new file mode 100644 index 0000000000..d518d39f2c --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_next_verify_prep.py @@ -0,0 +1,445 @@ +import pytest +import torch + +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +from sglang.kernels.ops.speculative.eagle import ( + prepare_draft_extend_inputs, + prepare_draft_extend_lengths, + prepare_verify_commit_outputs, +) + + +@pytest.mark.parametrize("bs", [0, 1, 3, 128, 256]) +@pytest.mark.parametrize("width", [1, 4, 7]) +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +def test_prepare_integer_outputs(bs, width, dtype): + depth = min(width, 4) + accepted = torch.ones(bs * 2, device="cuda", dtype=dtype)[::2] + seq = torch.arange(bs * 2, device="cuda", dtype=dtype)[::2] + 8192 + predict = torch.arange(bs * width * 2, device="cuda", dtype=dtype)[::2] + storage = torch.zeros((bs, depth * 2), device="cuda", dtype=torch.int64) + indices = storage[:, ::2] + if bs: + indices.copy_( + torch.randperm(bs * width, device="cuda").reshape(bs, width)[:, :depth] + ) + indices[0, -1] = -1 + prepare_verify_commit_outputs(predict, indices, accepted, seq) + prepare_draft_extend_inputs(accepted, predict, width) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = prepare_verify_commit_outputs(predict, indices, accepted, seq) + draft = prepare_draft_extend_inputs(accepted, predict, width) + for replay in range(depth + 1): + accepted.fill_(1 + replay % depth) + seq.add_(1) + predict.add_(17) + graph.replay() + expected_bonus = predict[indices][ + torch.arange(bs, device="cuda"), accepted.long() - 1 + ].int() + references = ( + seq + accepted, + expected_bonus, + accepted - 1, + torch.arange(bs, device="cuda") * width + accepted - 1, + predict.long(), + ) + for actual, expected in zip(outputs + draft, references): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("bs", [0, 1, 128, 257]) +@pytest.mark.parametrize("front", [0, 3]) +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +def test_draft_extend_lengths(bs, front, dtype): + seq = torch.arange(bs * 2, device="cuda", dtype=dtype)[::2] + outputs = prepare_draft_extend_lengths(seq, 4, front) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = prepare_draft_extend_lengths(seq, 4, front) + for delta in [0, 1, 8192]: + seq.add_(delta) + graph.replay() + expected = ( + (seq - front).clamp(min=0).int(), + torch.full((bs,), 4 + front, dtype=torch.int32, device="cuda"), + seq + 4, + ) + for a, b in zip(outputs, expected): + torch.testing.assert_close(a, b, rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("mixed", [False, True]) +def test_spec_mrope_positions_graph(dtype, mixed): + from types import SimpleNamespace + + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + positions = torch.arange(24, device="cuda", dtype=dtype)[::2] + mm_inputs = [None, None, None] + if mixed: + mm_inputs[1] = SimpleNamespace(mrope_position_delta=torch.tensor([[7]])) + batch = SimpleNamespace( + multimodal_inputs=mm_inputs, spec_info=SimpleNamespace(positions=positions) + ) + runner = SimpleNamespace(device="cuda") + output = SimpleNamespace(seq_lens=torch.ones(3, device="cuda")) + ForwardBatch.compute_spec_mrope_positions(output, runner, batch) + graph = torch.cuda.CUDAGraph() + if not mixed: + with torch.cuda.graph(graph): + ForwardBatch.compute_spec_mrope_positions(output, runner, batch) + for delta in [0, 17]: + positions.add_(delta) + if mixed: + ForwardBatch.compute_spec_mrope_positions(output, runner, batch) + else: + graph.replay() + offsets = torch.tensor([0, 7 if mixed else 0, 0], device="cuda").view(3, 1) + expected = (positions.view(3, 4) + offsets).flatten().unsqueeze(0).repeat(3, 1) + torch.testing.assert_close(output.mrope_positions, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("bs", [0, 1, 3, 129]) +@pytest.mark.parametrize("width", [1, 4, 7]) +@pytest.mark.parametrize("mrope", [False, True]) +def test_draft_extend_layout_graph(bs, width, mrope): + from sglang.kernels.ops.attention.position import compute_position_triton + from sglang.kernels.ops.speculative.eagle import prepare_draft_extend_layout + + seq = torch.arange(bs * 2, device="cuda", dtype=torch.int64)[::2] + prepare_draft_extend_layout(seq, width, mrope) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + prefix, lengths, post, info = prepare_draft_extend_layout(seq, width, mrope) + for value in [0, 1, 131000]: + seq.fill_(value) + graph.replay() + torch.testing.assert_close(prefix, seq.clamp(min=0).int(), rtol=0, atol=0) + torch.testing.assert_close( + lengths, torch.full_like(lengths, width), rtol=0, atol=0 + ) + torch.testing.assert_close(post, seq + width, rtol=0, atol=0) + if bs: + positions, starts = compute_position_triton( + seq.clamp(min=0).int(), torch.full_like(lengths, width), bs * width + ) + torch.testing.assert_close(info[0], positions, rtol=0, atol=0) + torch.testing.assert_close(info[1], starts, rtol=0, atol=0) + else: + assert info[0].numel() == info[1].numel() == 0 + if mrope: + torch.testing.assert_close( + info[2], info[0].unsqueeze(0).repeat(3, 1), rtol=0, atol=0 + ) + else: + assert info[2] is None + + +@pytest.mark.parametrize("bs", [1, 3, 128, 256]) +@pytest.mark.parametrize("depth", [1, 3, 7]) +@pytest.mark.parametrize("preallocated", [False, True]) +def test_chain_tree_graph(bs, depth, preallocated): + from sglang.kernels.ops.speculative.eagle import build_chain_tree + from sglang.srt.speculative.eagle_utils import ( + TreeMaskMode, + build_tree_kernel_efficient, + ) + + bonus = torch.arange(bs * 2, device="cuda", dtype=torch.int32)[::2] + draft = torch.arange(bs * depth * 4, device="cuda", dtype=torch.int64).reshape( + bs * 2, depth * 2 + )[::2, ::2] + seq = torch.arange(bs * 2, device="cuda", dtype=torch.int64)[::2] + parents = ( + torch.arange(-1, depth - 1, device="cuda").repeat(bs, 1) + if depth > 1 + else torch.empty((bs, 0), device="cuda", dtype=torch.int64) + ) + selected = torch.arange(depth, device="cuda").repeat(bs, 1) + width = depth + 1 + buf = ( + torch.empty((bs * width * width + 19,), dtype=torch.bool, device="cuda") + if preallocated + else None + ) + build_chain_tree(bonus, draft, seq, buf) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = build_chain_tree(bonus, draft, seq, buf) + for length in [0, 8192, 131000]: + seq.fill_(length) + bonus.add_(1) + draft.add_(3) + if buf is not None: + buf.fill_(False) + graph.replay() + expected = build_tree_kernel_efficient( + bonus, + parents, + selected, + draft, + seq.contiguous(), + bs * length, + 1, + depth, + width, + TreeMaskMode.QLEN_ONLY, + ) + for i, (a, e) in enumerate(zip(actual, expected)): + if i == 0: + a = a[: bs * width * width] + torch.testing.assert_close(a, e, rtol=0, atol=0) + if buf is not None: + assert not buf[-19:].any() + + +@pytest.mark.parametrize("bs", [0, 1, 3, 128]) +@pytest.mark.parametrize("depth", [1, 3, 7]) +@pytest.mark.parametrize("mrope", [False, True]) +def test_chain_tree_prepared_outputs_graph(bs, depth, mrope): + from sglang.kernels.ops.speculative.eagle import build_chain_tree + + table = torch.arange((bs + 3) * 128, dtype=torch.int32, device="cuda").reshape( + bs + 3, 128 + )[:, ::2] + requests = torch.arange(bs * 2, device="cuda")[::2] % (bs + 3) + seq = torch.arange(bs * 2, device="cuda")[::2] % 48 + bonus = torch.arange(bs, device="cuda") + draft = torch.arange(bs * depth, device="cuda").reshape(bs, depth) + + def run(): + return build_chain_tree( + bonus, + draft, + seq, + req_pool_indices=requests, + req_to_token=table, + with_mrope=mrope, + ) + + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + for i in range(3): + requests.add_(1).remainder_(bs + 3) + seq.add_(1).remainder_(48) + table.add_(13) + graph.replay() + positions = seq[:, None] + torch.arange(depth + 1, device="cuda") + expected = table[requests[:, None], positions].flatten().long() + torch.testing.assert_close(out[6], expected, rtol=0, atol=0) + torch.testing.assert_close(out[1], positions.flatten(), rtol=0, atol=0) + if mrope: + torch.testing.assert_close( + out[7], positions.flatten()[None, :].repeat(3, 1), rtol=0, atol=0 + ) + else: + assert out[7] is None + + +@pytest.mark.parametrize("bs", [1, 3, 128]) +@pytest.mark.parametrize("steps", [1, 3, 7]) +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("mrope", [False, True]) +def test_draft_cache_positions_graph(bs, steps, dtype, mrope): + from sglang.kernels.ops.speculative.cache_locs import ( + assign_draft_cache_locs_contiguous, + ) + + table = torch.arange((bs + 3) * 64, dtype=torch.int32, device="cuda").reshape( + bs + 3, 64 + ) + requests = torch.arange(bs, device="cuda") + seq = torch.arange(bs, device="cuda", dtype=dtype) % 48 + cache = torch.empty((bs * steps,), device="cuda", dtype=torch.int64) + positions = torch.empty_like(seq) + rope = torch.empty((3, bs), device="cuda", dtype=torch.int64) if mrope else None + + def run(): + assign_draft_cache_locs_contiguous[(bs,)]( + requests, + table, + seq, + cache, + 64, + 1, + steps, + positions=positions, + mrope=rope, + BS=bs, + WRITE_POSITIONS=True, + WRITE_MROPE=mrope, + ) + + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for i in range(3): + requests.add_(1).remainder_(bs + 3) + seq.add_(1).remainder_(48) + table.add_(17) + graph.replay() + expected = ( + table[ + requests[:, None], + seq[:, None].long() + torch.arange(steps, device="cuda"), + ] + .flatten() + .long() + ) + torch.testing.assert_close(cache, expected, rtol=0, atol=0) + torch.testing.assert_close(positions, seq, rtol=0, atol=0) + if mrope: + torch.testing.assert_close( + rope, seq.long()[None, :].repeat(3, 1), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("bs", [0, 1, 7, 128, 257]) +@pytest.mark.parametrize("width", [1, 4, 7]) +@pytest.mark.parametrize("interval", [0, 16, 512]) +def test_combined_verify_commit_metadata(bs, width, interval): + seq = torch.arange(bs * 2, device="cuda", dtype=torch.int64)[::2] + 509 + accepted = torch.ones(bs * 2, device="cuda", dtype=torch.int32)[::2] + predict = torch.arange(bs * width * 2, device="cuda", dtype=torch.int32)[::2] + accept = torch.arange(bs * width, device="cuda", dtype=torch.int64).reshape( + bs, width + ) + outputs = prepare_verify_commit_outputs( + predict, accept, accepted, seq, width, interval + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = prepare_verify_commit_outputs( + predict, accept, accepted, seq, width, interval + ) + for n in range(1, width + 1): + accepted.fill_(n) + seq.add_(1) + predict.add_(13) + graph.replay() + new_seq, bonus, (last, tracking), draft = outputs + rows = torch.arange(bs, device="cuda") + torch.testing.assert_close(new_seq, seq + accepted, rtol=0, atol=0) + torch.testing.assert_close( + bonus, predict[accept[rows, n - 1]].int(), rtol=0, atol=0 + ) + torch.testing.assert_close( + last, accept[rows, n - 1] - rows * width, rtol=0, atol=0 + ) + references = (accepted - 1, rows * width + n - 1, predict.long()) + for actual, expected in zip(draft, references): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + if interval: + crossed = seq // interval != (seq + accepted) // interval + offsets = (((seq + accepted) // interval) * interval - seq - 1).clamp(min=0) + ref = torch.where(crossed, accept[rows, offsets] - rows * width, -1) + torch.testing.assert_close(tracking, ref, rtol=0, atol=0) + else: + assert tracking is None + + +@pytest.mark.parametrize( + "operation", + ["cache", "commit", "commit_basic", "inputs", "lengths", "layout", "tree"], +) +def test_dynamic_batch_reuses_kernel(operation, monkeypatch): + """Changing request counts must not create new eager preparation variants.""" + from collections import defaultdict + + from sglang.kernels.ops.speculative import cache_locs, eagle + + kernels = { + "cache": cache_locs.assign_draft_cache_locs_contiguous, + "commit": eagle._verify_commit_outputs_kernel, + "commit_basic": eagle._verify_commit_outputs_kernel, + "inputs": eagle._draft_extend_inputs_kernel, + "lengths": eagle._prepare_draft_extend_lengths_kernel, + "layout": eagle._prepare_draft_extend_lengths_kernel, + "tree": eagle._build_chain_tree_kernel, + } + kernel = kernels[operation] + monkeypatch.setattr(kernel, "device_caches", defaultdict(kernel.create_binder)) + cache = kernel.device_caches[torch.cuda.current_device()][0] + width = 4 + max_bs = 257 + requests = torch.arange(max_bs, device="cuda") + seq = torch.full((max_bs,), 17, device="cuda", dtype=torch.int64) + accepted = torch.full((max_bs,), 3, device="cuda", dtype=torch.int32) + predict = torch.arange(max_bs * width + 3, device="cuda", dtype=torch.int32) + indices = torch.arange(max_bs * width, device="cuda").reshape(max_bs, width) + draft = torch.arange(max_bs * (width - 1), device="cuda").reshape(max_bs, width - 1) + table = torch.arange(max_bs * 64, device="cuda", dtype=torch.int32).reshape( + max_bs, 64 + ) + sizes = [1, 2, 15, 16, 17, 31, 32, 33, 127, 128, 129, 257, 257] + for i, bs in enumerate(sizes): + token_count = bs * width + (3 if i == len(sizes) - 1 else 0) + if operation == "cache": + locations = torch.empty((bs * 3,), device="cuda", dtype=torch.int64) + positions = torch.empty_like(seq[:bs]) + mrope = torch.empty((3, bs), device="cuda", dtype=torch.int64) + kernel[(bs,)]( + requests[:bs], + table, + seq[:bs], + locations, + 64, + 1, + 3, + positions=positions, + mrope=mrope, + BS=bs, + WRITE_POSITIONS=True, + WRITE_MROPE=True, + ) + torch.testing.assert_close(positions, seq[:bs], rtol=0, atol=0) + torch.testing.assert_close( + mrope, seq[:bs][None, :].expand(3, -1), rtol=0, atol=0 + ) + elif operation in ("commit", "commit_basic"): + out = eagle.prepare_verify_commit_outputs( + predict[:token_count], + indices[:bs], + accepted[:bs], + seq[:bs], + num_draft_tokens=width if operation == "commit" else 0, + mamba_track_interval=16 if operation == "commit" else 0, + ) + torch.testing.assert_close(out[0], seq[:bs] + accepted[:bs], rtol=0, atol=0) + elif operation == "inputs": + out = eagle.prepare_draft_extend_inputs( + accepted[:bs], predict[:token_count], width + ) + torch.testing.assert_close( + out[2], predict[:token_count].long(), rtol=0, atol=0 + ) + elif operation == "lengths": + eagle.prepare_draft_extend_lengths(seq[:bs], width, 3) + elif operation == "layout": + eagle.prepare_draft_extend_layout(seq[:bs], width, True) + else: + eagle.build_chain_tree( + predict[:bs], + draft[:bs], + seq[:bs], + req_pool_indices=requests[:bs], + req_to_token=table, + with_mrope=True, + ) + torch.cuda.synchronize() + assert len(cache) == 1, (operation, bs, token_count, len(cache)) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__]))