From 603a05e6c01b7db1fae29c61a46e2b6fd73a7d82 Mon Sep 17 00:00:00 2001 From: jiaqiang-dot-liu Date: Tue, 13 Oct 2026 14:17:49 +0800 Subject: [PATCH] [ROCm][Spec] Route NGRAM tree reconstruction to the JIT kernel on HIP (#43328) Co-authored-by: Cursor Co-authored-by: Thomas Wang --- python/sglang/srt/speculative/ngram_worker.py | 4 ++-- .../kernels/ops/speculative/test_ngram_utils.py | 9 +++------ test/registered/spec/test_spec_ngram_extra.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/speculative/ngram_worker.py b/python/sglang/srt/speculative/ngram_worker.py index e48f8fc7a5..cf6a237746 100644 --- a/python/sglang/srt/speculative/ngram_worker.py +++ b/python/sglang/srt/speculative/ngram_worker.py @@ -31,7 +31,7 @@ from sglang.srt.speculative.spec_utils import ( prepare_mamba_track_for_verify, record_stream_for_v2_verify, ) -from sglang.srt.utils import is_cpu, is_cuda, is_xpu +from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_xpu from sglang.srt.utils.async_probe import maybe_detect_inf, maybe_detect_nan _is_cpu = is_cpu() @@ -40,7 +40,7 @@ if is_xpu(): from sglang.kernels.ops.speculative.reconstruct_tree import ( reconstruct_indices_from_tree_mask_triton as reconstruct_indices_from_tree_mask, ) -elif is_cuda(): +elif is_cuda() or is_hip(): from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask else: from sgl_kernel.speculative import reconstruct_indices_from_tree_mask diff --git a/test/registered/kernels/ops/speculative/test_ngram_utils.py b/test/registered/kernels/ops/speculative/test_ngram_utils.py index a6fc76bdcc..d590427eb7 100644 --- a/test/registered/kernels/ops/speculative/test_ngram_utils.py +++ b/test/registered/kernels/ops/speculative/test_ngram_utils.py @@ -3,14 +3,11 @@ import sys import pytest import torch -from sglang.test.ci.ci_register import register_cuda_ci - -if torch.version.cuda is not None: - from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask -else: - from sgl_kernel import reconstruct_indices_from_tree_mask +from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_amd_ci(est_time=15, stage="jit-kernel-unit", runner_config="amd") def test_reconstruct_indices_from_tree_mask(): diff --git a/test/registered/spec/test_spec_ngram_extra.py b/test/registered/spec/test_spec_ngram_extra.py index 8a5f7096e3..ae68472363 100644 --- a/test/registered/spec/test_spec_ngram_extra.py +++ b/test/registered/spec/test_spec_ngram_extra.py @@ -2,19 +2,28 @@ import unittest import requests -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.ngram_fixture import NgramServerBase # Extra: Triton + Flashinfer NGRAM backends + non-overlap (sync V2) variant. # Sibling per-commit file (test_spec_ngram.py) keeps the Paged variant. register_cuda_ci(est_time=197, stage="extra-a", runner_config="1-gpu-large") +# AMD: flashinfer is not built in the ROCm sgl_kernel, so only the +# triton-backend class runs on ROCm. +register_amd_ci(est_time=130, suite="extra-a-test-1-gpu-large-amd") + +_AMD_SKIP_BACKEND = ( + "flashinfer attention backend is CUDA-only (not in the ROCm sgl_kernel build)" +) class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin): attention_backend = "triton" +@unittest.skipIf(is_hip(), _AMD_SKIP_BACKEND) class TestNgramSpeculativeDecodingNoOverlap(NgramServerBase, GSM8KMixin): """Non-overlap path: the scheduler drives the V2 worker synchronously (seq_lens advance via GenerationBatchResult.new_seq_lens, accepted tokens @@ -24,6 +33,7 @@ class TestNgramSpeculativeDecodingNoOverlap(NgramServerBase, GSM8KMixin): extra_args = ["--disable-overlap-schedule"] +@unittest.skipIf(is_hip(), _AMD_SKIP_BACKEND) class TestNgramSpeculativeDecodingFlashinfer(NgramServerBase, GSM8KMixin): attention_backend = "flashinfer" extra_args = ["--speculative-ngram-external-sam-budget", "8"]