diff --git a/python/sglang/kernels/ops/speculative/topk1.py b/python/sglang/kernels/ops/speculative/topk1.py index 93f658f2a1..03a6f91140 100644 --- a/python/sglang/kernels/ops/speculative/topk1.py +++ b/python/sglang/kernels/ops/speculative/topk1.py @@ -148,3 +148,15 @@ def draft_topk1_postprocess( num_warps=1, ) return topk_p, topk_index + + +def draft_topk1_argmax_only(next_token_logits: torch.Tensor): + """Select top-k=1 without mutating caller-owned positions.""" + # The upstream finalizer always advances positions. Draft-extend has already + # consumed its positions, so absorb that side effect in a discarded buffer. + scratch_positions = torch.zeros( + next_token_logits.shape[0], + dtype=torch.long, + device=next_token_logits.device, + ) + return draft_topk1_postprocess(next_token_logits, scratch_positions) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index e6506504f7..c2661bb481 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -5,7 +5,10 @@ from typing import List, Optional import torch -from sglang.kernels.ops.speculative.topk1 import draft_topk1_postprocess +from sglang.kernels.ops.speculative.topk1 import ( + draft_topk1_argmax_only, + draft_topk1_postprocess, +) from sglang.srt.configs.model_config import get_dsa_mtp_topk_width from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.graph_runner.eagle_draft_extend_npu_graph_runner import ( @@ -891,8 +894,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): ) draft_probs_list.append(probs) forward_batch.positions.add_(1) - elif self.topk == 1 and not _is_hip: - if _is_cuda: + elif self.topk == 1: + if _is_cuda or _is_hip: topk_p, topk_index = draft_topk1_postprocess( logits_output.next_token_logits, forward_batch.positions, @@ -1242,6 +1245,11 @@ class EagleDraftWorker(EagleDraftWorkerBase): batch.sampling_info.temperatures, batch.sampling_info.top_ks, ) + elif self.topk == 1 and _is_hip: + ret_topk_p, ret_topk_index = draft_topk1_argmax_only( + draft_logits_output.next_token_logits + ) + ret_draft_probs = None elif self.topk == 1 and not _is_hip: # Gated to CUDA: see #26358 — ROCm's argmax tie-break corrupts # MTP draft selection on FP8 logits. diff --git a/test/registered/kernels/ops/speculative/test_spec_topk1.py b/test/registered/kernels/ops/speculative/test_spec_topk1.py index b8c0c049b2..42b94986be 100644 --- a/test/registered/kernels/ops/speculative/test_spec_topk1.py +++ b/test/registered/kernels/ops/speculative/test_spec_topk1.py @@ -1,12 +1,16 @@ -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-small") +register_amd_ci(est_time=10, stage="stage-b", runner_config="1-gpu-small-amd") import unittest import torch -from sglang.kernels.ops.speculative.topk1 import draft_topk1_postprocess +from sglang.kernels.ops.speculative.topk1 import ( + draft_topk1_argmax_only, + draft_topk1_postprocess, +) from sglang.test.test_utils import CustomTestCase @@ -74,6 +78,56 @@ class TestSpecTopk1Triton(CustomTestCase): positions, expected_positions, rtol=0, atol=0 ) + def test_draft_topk1_masks_vocab_tail_and_nan(self): + for vocab_size in (151666, 248320): + cases = { + "all_nan": torch.full( + (2, vocab_size), + float("nan"), + dtype=torch.float32, + device=self.device, + ), + "all_neg_inf": torch.full( + (2, vocab_size), + -float("inf"), + dtype=torch.float32, + device=self.device, + ), + "all_negative": torch.full( + (2, vocab_size), + -1.0, + dtype=torch.float32, + device=self.device, + ), + } + partial_nan = ( + torch.arange(vocab_size, dtype=torch.float32, device=self.device) + .remainder(97) + .repeat(2, 1) + ) + partial_nan[:, vocab_size // 2] = float("nan") + cases["partial_nan"] = partial_nan + + for name, logits in cases.items(): + with self.subTest(vocab_size=vocab_size, case=name): + expected_logits = torch.where( + torch.isnan(logits), + torch.full_like(logits, -1e30), + logits, + ) + expected_index = torch.argmax(expected_logits, dim=-1, keepdim=True) + + topk_p, topk_index = draft_topk1_argmax_only(logits) + + torch.testing.assert_close( + topk_index, expected_index, rtol=0, atol=0 + ) + torch.testing.assert_close( + topk_p, torch.ones_like(topk_p), rtol=0, atol=0 + ) + self.assertTrue(torch.all(topk_index >= 0).item()) + self.assertTrue(torch.all(topk_index < vocab_size).item()) + def test_draft_topk1_postprocess_can_write_draft_token_column(self): batch_size = 17 # Multi-split vocab so the fused write composes with the split reduction.