[AMD] [GLM-5.3-Flash] Enable FP8 and MXFP4 serving on gfx950 (#39273)

Co-authored-by: Byron Hsu <byron+per@periodiclabs.ai>
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
Co-authored-by: long10024070 <long10024070@users.noreply.github.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: sunxxuns <126995791+sunxxuns@users.noreply.github.com>
Co-authored-by: Long Luong <long.luong2+personal@onemount.com>
This commit is contained in:
sonle5
2026-10-01 15:59:44 -07:00
committed by GitHub
co-authored by Byron Hsu Cheng Wan long10024070 Kevin Mi Kevin Mi Claude Opus 5.5 sunxxuns Long Luong
parent 80bb3fb651
commit f17f7705a5
34 changed files with 1045 additions and 115 deletions
@@ -1166,6 +1166,30 @@ class OpenAIServingChat(OpenAIServingBase):
return "text", processed_messages.prompt_ids
return "input_ids", processed_messages.prompt_ids
def _can_reuse_text_only_prompt_ids(
self, processed_messages: MessageProcessingResult, is_multimodal: bool
) -> bool:
# Moss-VL invokes its processor for text-only requests, and that processor
# requires the rendered text rather than pre-tokenized ids.
is_moss_vl = (
"MossVLForConditionalGeneration"
in self.tokenizer_manager.model_config.hf_config.architectures
)
return (
is_multimodal
and not is_moss_vl
and self.chat_encoding_spec is None
and self.template_manager.chat_template_name is None
and not self._prompt_text_round_trip_is_lossy
and not self._tokenizer_auto_adds_specials
and isinstance(processed_messages.prompt_ids, list)
and bool(processed_messages.prompt_ids)
and not processed_messages.image_data
and not processed_messages.video_data
and not processed_messages.audio_data
and not processed_messages.modalities
)
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
@@ -1233,7 +1257,9 @@ class OpenAIServingChat(OpenAIServingBase):
)
# Handle single vs multiple requests
if request.input_ids is not None:
if request.input_ids is not None or self._can_reuse_text_only_prompt_ids(
processed_messages, is_multimodal
):
prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
else:
prompt_key, prompt_value = self._engine_prompt(
@@ -1337,8 +1363,12 @@ class OpenAIServingChat(OpenAIServingBase):
tool_call_constraint = None
effective_tools = self._effective_tools(request)
glm_constraint = self.tool_call_parser == "glm47" and not any(
tool.function.strict for tool in effective_tools
# Only tool-bearing requests get the full-assistant EBNF: its terminal
# state finishes a request even under ignore_eos.
glm_constraint = (
self.tool_call_parser == "glm47"
and bool(effective_tools)
and not any(tool.function.strict for tool in effective_tools)
)
if glm_constraint:
enable_thinking = (request.chat_template_kwargs or {}).get(
@@ -23,7 +23,6 @@ from sglang.srt.layers.attention.dsa.utils import (
dsa_use_prefill_cp,
)
from sglang.srt.layers.attention.mqa_logits_utils import (
MQA_LOGITS_BYTES_PER_ELEM,
MQA_LOGITS_MAX_BYTES_ROCM,
mqa_logits_budget_bytes,
mqa_logits_row_bytes,
@@ -901,7 +900,8 @@ class IndexerKPool(MultiPlatformOp):
num_q, num_k = q_fp8.shape[0], k_fp8.shape[0]
rows_per_call = mqa_logits_rows_per_chunk(
num_rows=num_q,
row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM,
# aiter pads each output row to 256 columns
row_bytes=mqa_logits_row_bytes(num_k),
budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM,
)
if rows_per_call is None:
@@ -122,7 +122,7 @@ def _gather_index_k_scale_prefix_into_kernel(
token_id = tl.program_id(0)
page_idx = token_id // PAGE_SIZE
token_offset_in_page = token_id % PAGE_SIZE
page = tl.load(page_indices_ptr + page_idx)
page = tl.load(page_indices_ptr + page_idx).to(tl.int64)
offs = tl.arange(0, BLOCK_D)
mask = offs < HEAD_DIM
@@ -44,10 +44,19 @@ class DsaGraphVariants:
def create_attention_graph_variants(hf_config) -> Optional[AttentionGraphVariants]:
from sglang.srt.configs.model_config import get_dsa_index_topk, is_deepseek_dsa
from sglang.srt.configs.model_config import (
get_dsa_index_kpool,
get_dsa_index_topk,
is_deepseek_dsa,
)
from sglang.srt.utils import is_hip
if is_hip() and is_deepseek_dsa(hf_config):
# KPool has no dense-skip path: both variants run the full indexer.
# Capture it once, keeping the regular DSA dense/sparse split intact.
if get_dsa_index_kpool(hf_config) > 1:
return None
index_topk = get_dsa_index_topk(hf_config)
logger.info(
"[dense-decode] DSA dual-graph enabled: capturing "
@@ -20,7 +20,7 @@ from sglang.srt.layers.attention.linear.utils import (
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.utils import is_cpu, is_cuda, is_npu
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_npu
from sglang.srt.utils.common import is_gfx95_supported, rank0_log
# KDA always uses the triton causal_conv1d_fn (no CUDA override).
@@ -1285,6 +1285,16 @@ class KDAAttnBackend(MambaAttnBackendBase):
# other batch/architecture combinations are measured. The snapshot
# path and the separate CuTe path are unchanged.
return False
if is_hip() and not (
is_gfx95_supported()
and 1 <= batch_size <= 16
and draft_token_num in (6, 8)
and mixed_qkv.dtype == torch.bfloat16
and layer.conv_weights.dtype == torch.float32
):
# measured gfx950 wins only: larger batches lose the launch saving to
# duplicated convolution work
return False
expected_dim = (
2 * layer.num_q_heads * layer.head_k_dim
+ layer.num_v_heads * layer.head_v_dim
+23 -2
View File
@@ -952,6 +952,9 @@ class ReqKvInfo:
mamba_last_track_idx: Optional[int] = None # 0 or 1
# Seq len of the last cached mamba state
mamba_last_track_seqlen: Optional[int] = None
# Seq len of the other ping-pong slot's state. None means what is in
# that slot cannot be named (never written, donated, freed).
mamba_prev_track_seqlen: Optional[int] = None
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
mamba_cow_src_index: Optional[torch.Tensor] = None
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
@@ -1475,7 +1478,14 @@ class Req(ReqDllmMixin):
# overallocated range and are reclaimed by release_kv_cache. #22373.
if get_serving().strip_thinking_cache and self.reasoning_tokens > 0:
return min(self.kv.kv_committed_len, len(self.origin_input_ids))
return self.kv.kv_committed_len
if self.finished_len is None:
return self.kv.kv_committed_len
# A verify step commits a whole accepted chunk, so output_ids can run
# past the stop; the client never sees those tokens, so nothing matches.
return min(
self.kv.kv_committed_len,
len(self.origin_input_ids) + len(self.output_ids_through_stop),
)
def update_spec_correct_drafts_histogram(self, num_correct_drafts: int):
"""Record one step accepted draft count (excludes bonus token) into the histogram."""
@@ -1988,6 +1998,7 @@ class Req(ReqDllmMixin):
self.kv.mamba_next_track_idx = None
self.kv.mamba_last_track_idx = None
self.kv.mamba_last_track_seqlen = None
self.kv.mamba_prev_track_seqlen = None
self.mamba_branching_seqlen = None
self.kv.mamba_cow_src_index = None
self.kv.mamba_needs_clear = False
@@ -2212,7 +2223,8 @@ def set_mamba_track_indices_from_reqs(
if track_positions is None:
# Guard: mamba_next_track_idx may be None for requests that haven't
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
# Default to 0 (first ping-pong slot) to avoid TypeError.
# Keep the gather in bounds for those requests. Freed rows are
# invalidated below so they cannot scatter into a recycled slot.
track_positions = [
(
req.kv.mamba_next_track_idx
@@ -2221,6 +2233,9 @@ def set_mamba_track_indices_from_reqs(
)
for req in batch.reqs
]
freed_rows = [
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
]
batch.mamba_track_buffer_indices = list(track_positions)
idx = (
torch.tensor(
@@ -2234,6 +2249,12 @@ def set_mamba_track_indices_from_reqs(
batch.mamba_track_indices = (
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
)
if freed_rows:
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
# been freed and reused by another request. The downstream scatter
# treats a negative destination as a no-op, while the old fallback to
# position 0 could overwrite the reused live slot.
batch.mamba_track_indices[freed_rows] = -1
def release_req(
@@ -648,7 +648,12 @@ class PrefillAdder:
self.rem_chunk_tokens = rem_chunk_tokens
self.chunked_req_limit: Optional[int] = None
self.dllm_config = dllm_config
self.exact_chunk_fill = _use_exact_chunk_fill() and dllm_config is None
# Mamba checkpoints only land on page-aligned chunk ends.
self.exact_chunk_fill = (
_use_exact_chunk_fill()
and dllm_config is None
and not tree_cache.supports_mamba()
)
if self.dllm_config is not None:
self._init_dllm_meta(dllm_config)
@@ -1366,6 +1366,9 @@ class SchedulerBatchResultProcessor:
if completed_mamba_boundary and not lazy:
req.kv.mamba_last_track_idx = batch.mamba_track_buffer_indices[i]
# The slot that stops being the latest still holds its
# checkpoint; name it so a short key can fall back to it.
req.kv.mamba_prev_track_seqlen = req.kv.mamba_last_track_seqlen
req.kv.mamba_last_track_seqlen = req.kv.kv_committed_len - lookahead
elif (
req.finished()
@@ -1495,13 +1498,17 @@ class SchedulerBatchResultProcessor:
track_idx = req.kv.mamba_next_track_idx
if not known_boundary and batch.mamba_track_buffer_indices is not None:
track_idx = batch.mamba_track_buffer_indices[i]
previous_track_seqlen = req.kv.mamba_last_track_seqlen
if not known_boundary:
req.kv.mamba_last_track_seqlen = track_seqlen
if lazy:
# Lazy frees the slot it stops tracking, so nothing names the
# previous checkpoint there; mamba_prev_track_seqlen stays None.
self.mamba_lazy_post_decode_at_boundary(req, batch, track_idx)
else:
if not known_boundary:
req.kv.mamba_last_track_idx = track_idx
req.kv.mamba_prev_track_seqlen = previous_track_seqlen
req.kv.mamba_next_track_idx = (
batch.req_to_token_pool.get_mamba_ping_pong_other_idx(track_idx)
)
@@ -81,6 +81,8 @@ class InsertParams:
# Mamba specific
mamba_value: Optional[torch.Tensor] = None
# The ping-pong slot prepare picked; cleanup keeps that same slot.
mamba_keep_idx: Optional[int] = None
# DSV4 NPU C128 sidecar pages, one page id per physical C128 page group.
c128_value: Optional[torch.Tensor] = None
+6 -1
View File
@@ -346,7 +346,12 @@ def _release_overallocated_kv_indices(
# strip_thinking_cache intentionally reports output tokens as overallocated
# so they fall into the free path below (#22373).
if spec_algo is None and not get_serving().strip_thinking_cache:
assert start_p == end_p, (
# A stop landing before the last committed token does the same, via
# effective_kv_committed_len().
assert start_p == end_p or (
req.finished_len is not None
and len(req.origin_input_ids) + req.finished_len < req.kv.kv_committed_len
), (
f"Unexpected overallocated KV cache, {req.kv.kv_committed_len=}, {req.kv.kv_allocated_len=}"
)
@@ -1620,6 +1620,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
)
buf[:n] = slots
req.kv.mamba_ping_pong_track_buffer = buf
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = (
0
@@ -1720,6 +1721,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_cow_src_index = None
req.kv.mamba_needs_clear = False
@@ -536,6 +536,28 @@ class MambaComponent(TreeComponent):
else:
self.cache.req_to_token_pool.mamba_allocator.free(mamba_value)
def _select_finished_checkpoint(
self, req: Req, token_ids_len: int
) -> Optional[tuple[int, int]]:
# None means donate nothing, not "no slot found".
pool = self.cache.req_to_token_pool
keep_idx = pool.get_mamba_ping_pong_keep_idx(req)
cache_len = req.kv.mamba_last_track_seqlen or 0
if cache_len <= token_ids_len:
return cache_len, keep_idx
# Overshoot: the latest state ran past the key. The other slot always
# holds a tensor; only this seqlen says whether a key can name it.
previous_cache_len = req.kv.mamba_prev_track_seqlen
if (
pool.mamba_ping_pong_track_buffer_size != 2
or previous_cache_len is None
or previous_cache_len > token_ids_len
):
return None
return previous_cache_len, pool.get_mamba_ping_pong_other_idx(keep_idx)
def prepare_for_caching_req(
self,
req: Req,
@@ -564,9 +586,11 @@ class MambaComponent(TreeComponent):
if cache_len is None:
cache_len = 0
if self.cache.enable_mamba_extra_buffer:
keep_idx = self.cache.req_to_token_pool.get_mamba_ping_pong_keep_idx(
req
)
checkpoint = self._select_finished_checkpoint(req, token_ids_len)
if checkpoint is None:
return 0
cache_len, keep_idx = checkpoint
insert_params.mamba_keep_idx = keep_idx
active_value = (
req.kv.mamba_ping_pong_track_buffer[keep_idx].unsqueeze(-1).clone()
)
@@ -639,11 +663,11 @@ class MambaComponent(TreeComponent):
return
if self.cache.enable_mamba_extra_buffer:
keep_idx = (
pool.get_mamba_ping_pong_keep_idx(req)
if mamba_value_inserted
else None
# Keep the slot prepare picked, so cleanup cannot pick another.
prepared_keep_idx = (
insert_params.mamba_keep_idx if insert_params is not None else None
)
keep_idx = prepared_keep_idx if mamba_value_inserted else None
pool.free_mamba_cache(
req, mamba_ping_pong_track_buffer_to_keep=keep_idx
)
@@ -657,6 +681,7 @@ class MambaComponent(TreeComponent):
):
self._free_mamba_value(insert_params.mamba_value)
req.kv.mamba_last_track_seqlen = None
req.kv.mamba_prev_track_seqlen = None
def build_external_linker_transfer(
self,
@@ -526,8 +526,8 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
declared is SharedReadEnds.IN_REPLAY
and self.in_graph_metadata_prep_done is None
):
# TODO: this lands EARLIER than declared; POST_REPLAY is the sound one.
return SharedReadEnds.PRE_REPLAY
# no in-graph marker (e.g. HIP): only a post-replay event covers its reads
return SharedReadEnds.POST_REPLAY
return declared
def _publish_read_done(self, in_graph: bool):
@@ -638,9 +638,7 @@ class DeepseekMHAForwardMixin:
Returns: (kv_a, k_pe) both in BF16
"""
backend = get_attn_backend()
if isinstance(backend, TboAttnBackend): # if enable tbo, get primary backend
backend = backend.primary
backend = resolve_attn_backend(forward_batch)
kv_indices = backend.forward_metadata.page_table_1_flattened
assert kv_indices is not None, (
"page_table_1_flattened should have been generated for FP8 MHA path"
+9 -3
View File
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL_OPS,
MHCState,
@@ -317,11 +318,14 @@ class Glm5NextVisionModel(GlmOcrVisionModel):
swiglu_limit=vision_config.swiglu_limit,
)
self.downsample = nn.Conv2d(
# These non-overlapping patches are equivalent to unfold + linear and
# avoid MIOpen's expensive per-shape convolution search on ROCm.
self.downsample = Conv2dLayer(
in_channels=vision_config.hidden_size,
out_channels=vision_config.out_hidden_size,
kernel_size=vision_config.spatial_merge_size,
stride=vision_config.spatial_merge_size,
disable_linear=False,
)
self.post_layernorm = GlmOcrRMSNorm(
vision_config.hidden_size, eps=vision_config.rms_norm_eps
@@ -1596,8 +1600,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
params_dict = dict(self.named_parameters())
def maybe_map_fp8_block_scale_name(name: str) -> str:
if name.endswith("weight_scale"):
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
# Quark stores dequantization scales without native block-FP8's
# "_inv" suffix, including fused w13/w2 expert parameters.
if name not in params_dict and name.endswith("weight_scale"):
candidate = name + "_inv"
if candidate in params_dict:
return candidate
return name
@@ -59,6 +59,8 @@ from sglang.srt.utils.device_timer import device_timer_ctx
if TYPE_CHECKING:
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker
_GREEDY_VARIANT = "greedy"
@dataclass
class EagleDraftInputBuffers(ForwardInputBuffers):
@@ -307,8 +309,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
return torch.int64
def _make_graph_key(self, bs, stream_idx=None, variant_label=None):
# EAGLE doesn't use stream_idx / lora variants.
return ShapeKey(size=bs)
# EAGLE doesn't use stream_idx; variant_label is _GREEDY_VARIANT or None.
return ShapeKey(size=bs, variant_label=variant_label)
def _sampling_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
# Under rejection sampling an all-greedy batch replays the graph that
# skips the draft proposal distribution (see draft_forward).
if (
get_spec().speculative_use_rejection_sampling
and forward_batch.sampling_info is not None
and forward_batch.sampling_info.is_all_greedy
):
return _GREEDY_VARIANT
return None
# -----------------------------------------------------------------
# can_run_graph
@@ -332,7 +345,13 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
cuda_graph_bs = forward_batch.batch_size
is_bs_supported = (
self.backend.can_run(forward_batch, self._make_graph_key(cuda_graph_bs))
self.backend.can_run(
forward_batch,
self._make_graph_key(
cuda_graph_bs,
variant_label=self._sampling_variant(forward_batch),
),
)
if self.disable_padding
else cuda_graph_bs <= self.max_bs
)
@@ -498,7 +517,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
# per-step forwards inside draft_forward must not re-plan.
forward_batch.mark_forward_metadata_ready()
self.deepep_adapter.capture(is_extend_in_batch=False)
shape_key = self._make_graph_key(num_seqs)
post_warmup_hook = getattr(
self.draft_attn_backend, "on_after_cuda_graph_warmup", None
)
@@ -508,12 +526,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
post_warmup_hook=post_warmup_hook,
run_lm_head=True,
)
self.backend.capture_one(
shape_key,
run_once,
capture_inputs=None,
post_warmup_hook=post_warmup_hook,
variants = (
(None, _GREEDY_VARIANT)
if get_spec().speculative_use_rejection_sampling
else (None,)
)
for variant in variants:
sampling_info.is_all_greedy = variant == _GREEDY_VARIANT
self.backend.capture_one(
self._make_graph_key(num_seqs, variant_label=variant),
run_once,
capture_inputs=None,
post_warmup_hook=post_warmup_hook,
)
def _postprocess_output_to_raw_bs(self, out, raw_bs):
parent_list, top_scores_index, draft_tokens, draft_probs = (
@@ -694,7 +719,9 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
self.bs = bs
# Replay via backend
shape_key = self._make_graph_key(bs)
shape_key = self._make_graph_key(
bs, variant_label=self._sampling_variant(forward_batch)
)
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft"):
out = self._replay_graph(shape_key, forward_batch)
if self.buffers.dsa_seed_topk is not None:
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Callable, Optional
import torch
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
deployment_attn_dp_size,
@@ -631,12 +632,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
)
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
# Snapshot built -- the forward is done reading the shared pool. Publish
# a read-done event the scheduler's WAR barrier waits on (draft extend
# is the EAGLE-family last shared-read phase; last write wins the mailbox).
read_done = self.device_module.Event()
read_done.record()
self.model_runner.shared_read_done_event = read_done
# Draft extend is the last shared reader. UNKNOWN must discard any
# earlier phase's event so the scheduler uses its whole-forward fence.
self.model_runner.shared_read_done_event = None
shared_read_ends = self.draft_extend_attn_backend.shared_read_ends(
self.forward_mode
)
if shared_read_ends is SharedReadEnds.PRE_REPLAY:
self._publish_read_done(in_graph=False)
self.raw_bs = raw_bs
self.bs = bs
@@ -644,6 +647,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft_extend"):
out = self._replay_graph(shape_key, forward_batch)
# This runner has no in-graph read-done marker. A later event is safe;
# an event before replay cannot cover metadata reads inside the graph.
if shared_read_ends in (SharedReadEnds.IN_REPLAY, SharedReadEnds.POST_REPLAY):
self._publish_read_done(in_graph=False)
out = LogitsProcessorOutput(
next_token_logits=out.next_token_logits[:raw_bs],
# CUDA graph replay reuses its captured output storage. These states
@@ -700,7 +700,13 @@ class EagleDraftWorker(EagleDraftWorkerBase):
score_list: List[torch.Tensor] = []
token_list: List[torch.Tensor] = []
parents_list: List[torch.Tensor] = []
if get_spec().speculative_use_rejection_sampling:
# Verify commits argmax for an all-greedy batch and never reads the
# draft distribution, so skip building it.
needs_draft_probs = (
get_spec().speculative_use_rejection_sampling
and not forward_batch.sampling_info.is_all_greedy
)
if needs_draft_probs:
draft_probs_list: List[torch.Tensor] = [spec_info.draft_probs]
topk1_chain_fits = (
@@ -715,7 +721,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
topk1_chain_fits
and _is_cuda
and self.hot_token_id is None
and not get_spec().speculative_use_rejection_sampling
and not needs_draft_probs
):
draft_tokens_topk1 = torch.empty(
(topk_index.shape[0], self.speculative_num_steps),
@@ -778,7 +784,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
maybe_detect_inf(
logits_output.next_token_logits, f"draft_forward step {i}"
)
if get_spec().speculative_use_rejection_sampling:
if needs_draft_probs:
probs, topk_p, topk_index = sample_draft_proposal(
logits_output.next_token_logits,
forward_batch.sampling_info.temperatures,
@@ -804,7 +810,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
probs = renorm_draft_probs(
logits_output.next_token_logits,
forward_batch.sampling_info,
get_spec().speculative_use_rejection_sampling,
needs_draft_probs,
)
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
forward_batch.positions.add_(1)
@@ -821,9 +827,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
hidden_states = logits_output.hidden_states
draft_probs = (
torch.stack(draft_probs_list, dim=1)
if get_spec().speculative_use_rejection_sampling
else None
torch.stack(draft_probs_list, dim=1) if needs_draft_probs else None
)
# Organize the results