[AMD] Skip full-vocab softmax in EAGLE topk==1 draft on ROCm (#35872)

Co-authored-by: chuyeh <chuyeh@users.noreply.github.com>
This commit is contained in:
jacky.cheng
2026-09-22 01:57:37 -07:00
committed by GitHub
co-authored by chuyeh
parent c2f14bfbc4
commit 790551c382
3 changed files with 79 additions and 5 deletions
@@ -148,3 +148,15 @@ def draft_topk1_postprocess(
num_warps=1,
)
return topk_p, topk_index
def draft_topk1_argmax_only(next_token_logits: torch.Tensor):
"""Select top-k=1 without mutating caller-owned positions."""
# The upstream finalizer always advances positions. Draft-extend has already
# consumed its positions, so absorb that side effect in a discarded buffer.
scratch_positions = torch.zeros(
next_token_logits.shape[0],
dtype=torch.long,
device=next_token_logits.device,
)
return draft_topk1_postprocess(next_token_logits, scratch_positions)
@@ -5,7 +5,10 @@ from typing import List, Optional
import torch
from sglang.kernels.ops.speculative.topk1 import draft_topk1_postprocess
from sglang.kernels.ops.speculative.topk1 import (
draft_topk1_argmax_only,
draft_topk1_postprocess,
)
from sglang.srt.configs.model_config import get_dsa_mtp_topk_width
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.graph_runner.eagle_draft_extend_npu_graph_runner import (
@@ -891,8 +894,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
)
draft_probs_list.append(probs)
forward_batch.positions.add_(1)
elif self.topk == 1 and not _is_hip:
if _is_cuda:
elif self.topk == 1:
if _is_cuda or _is_hip:
topk_p, topk_index = draft_topk1_postprocess(
logits_output.next_token_logits,
forward_batch.positions,
@@ -1242,6 +1245,11 @@ class EagleDraftWorker(EagleDraftWorkerBase):
batch.sampling_info.temperatures,
batch.sampling_info.top_ks,
)
elif self.topk == 1 and _is_hip:
ret_topk_p, ret_topk_index = draft_topk1_argmax_only(
draft_logits_output.next_token_logits
)
ret_draft_probs = None
elif self.topk == 1 and not _is_hip:
# Gated to CUDA: see #26358 — ROCm's argmax tie-break corrupts
# MTP draft selection on FP8 logits.