From 7a719e9a65e7f52b012ab37211b5f174e5d3d38d Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Mon, 5 Oct 2026 00:36:26 +0800 Subject: [PATCH] [AMD] Fix RoPE cache dtype and diffusion CI failures (#42494) --- .github/workflows/pr-test-amd.yml | 5 + .../kernels/aot/csrc/elementwise/pos_enc.cu | 109 ++++++++++-------- .../runtime/layers/quantization/__init__.py | 78 +++++++------ .../minimax_h3/stages/denoising.py | 3 +- .../unit/test_diffusion_import_isolation.py | 31 ++++- scripts/ci/amd/amd_ci_install_dependency.sh | 6 + test/registered/rotary/test_rope_rocm.py | 31 ++++- 7 files changed, 171 insertions(+), 92 deletions(-) diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index a2a1f98127..913c331604 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -990,6 +990,9 @@ jobs: - name: Run diffusion server tests (2-GPU) timeout-minutes: ${{ (inputs.runner_arch || 'mi300') == 'mi300' && 180 || 150 }} + env: + HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} run: | # AMD CI: All 2-GPU tests including LoRA # Tests: T2V, T2I, I2V, LoRA @@ -998,6 +1001,8 @@ jobs: # - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available) # - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings docker exec \ + -e HF_TOKEN \ + -e HUGGING_FACE_HUB_TOKEN \ -e SGLANG_E2E_TOLERANCE=0.3 \ -e SGLANG_STAGE_TIME_TOLERANCE=0.2 \ -e SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=0.6 \ diff --git a/python/sglang/kernels/aot/csrc/elementwise/pos_enc.cu b/python/sglang/kernels/aot/csrc/elementwise/pos_enc.cu index 9b8bbd1d54..a5d25bcc1b 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/pos_enc.cu +++ b/python/sglang/kernels/aot/csrc/elementwise/pos_enc.cu @@ -5,16 +5,19 @@ #include #include +#include + #include "utils.h" -template +template inline __device__ void apply_token_rotary_embedding( scalar_t* __restrict__ arr, - const scalar_t* __restrict__ cos_ptr, - const scalar_t* __restrict__ sin_ptr, + const cache_t* __restrict__ cos_ptr, + const cache_t* __restrict__ sin_ptr, int rot_offset, int embed_dim) { int x_index, y_index; + // Round cache values to the query dtype, as in the same-dtype cache path. scalar_t cos, sin; if (IS_NEOX) { // GPT-NeoX style rotary embedding. @@ -36,7 +39,7 @@ inline __device__ void apply_token_rotary_embedding( arr[y_index] = y * cos + x * sin; } -template +template inline __device__ void apply_rotary_embedding( scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, // head_size] or [num_tokens, num_heads, @@ -45,7 +48,7 @@ inline __device__ void apply_rotary_embedding( // [batch_size, seq_len, num_kv_heads, // head_size] or [num_tokens, num_kv_heads, // head_size] - const scalar_t* cache_ptr, + const cache_t* cache_ptr, const int head_size, const int num_heads, const int num_kv_heads, @@ -55,15 +58,16 @@ inline __device__ void apply_rotary_embedding( const int64_t key_stride, const int64_t head_stride) { const int embed_dim = rot_dim / 2; - const scalar_t* cos_ptr = cache_ptr; - const scalar_t* sin_ptr = cache_ptr + embed_dim; + const cache_t* cos_ptr = cache_ptr; + const cache_t* sin_ptr = cache_ptr + embed_dim; const int nq = num_heads * embed_dim; for (int i = threadIdx.x; i < nq; i += blockDim.x) { const int head_idx = i / embed_dim; const int64_t token_head = token_idx * query_stride + head_idx * head_stride; const int rot_offset = i % embed_dim; - apply_token_rotary_embedding(query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim); + apply_token_rotary_embedding( + query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim); } if (key != nullptr) { @@ -72,24 +76,25 @@ inline __device__ void apply_rotary_embedding( const int head_idx = i / embed_dim; const int64_t token_head = token_idx * key_stride + head_idx * head_stride; const int rot_offset = i % embed_dim; - apply_token_rotary_embedding(key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim); + apply_token_rotary_embedding( + key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim); } } } -template +template __global__ void rotary_embedding_kernel( - const int64_t* __restrict__ positions, // [batch_size, seq_len] or - // [num_tokens] - scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, - // head_size] or [num_tokens, num_heads, - // head_size] - scalar_t* __restrict__ key, // nullptr or - // [batch_size, seq_len, num_kv_heads, - // head_size] or [num_tokens, num_kv_heads, - // head_size] - const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // - // 2] + const int64_t* __restrict__ positions, // [batch_size, seq_len] or + // [num_tokens] + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, + // head_size] or [num_tokens, num_heads, + // head_size] + scalar_t* __restrict__ key, // nullptr or + // [batch_size, seq_len, num_kv_heads, + // head_size] or [num_tokens, num_kv_heads, + // head_size] + const cache_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // + // 2] const int rot_dim, const int64_t query_stride, const int64_t key_stride, @@ -100,9 +105,9 @@ __global__ void rotary_embedding_kernel( // Each thread block is responsible for one token. const int token_idx = blockIdx.x; int64_t pos = positions[token_idx]; - const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim; + const cache_t* cache_ptr = cos_sin_cache + pos * rot_dim; - apply_rotary_embedding( + apply_rotary_embedding( query, key, cache_ptr, @@ -177,32 +182,40 @@ void rotary_embedding( const at::cuda::OptionalCUDAGuard device_guard(device_of(query)); const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); DISPATCH_FLOAT_TYPES(query.scalar_type(), "rotary_embedding", [&] { - if (is_neox) { - rotary_embedding_kernel<<>>( - positions.data_ptr(), - query.data_ptr(), - key.has_value() ? key->data_ptr() : nullptr, - cos_sin_cache.data_ptr(), - rot_dim, - query_stride, - key_stride, - head_stride, - num_heads, - num_kv_heads, - head_size); + auto launch = [&](auto* cache_ptr) { + using cache_t = std::remove_pointer_t; + if (is_neox) { + rotary_embedding_kernel<<>>( + positions.data_ptr(), + query.data_ptr(), + key.has_value() ? key->data_ptr() : nullptr, + cache_ptr, + rot_dim, + query_stride, + key_stride, + head_stride, + num_heads, + num_kv_heads, + head_size); + } else { + rotary_embedding_kernel<<>>( + positions.data_ptr(), + query.data_ptr(), + key.has_value() ? key->data_ptr() : nullptr, + cache_ptr, + rot_dim, + query_stride, + key_stride, + head_stride, + num_heads, + num_kv_heads, + head_size); + } + }; + if (cos_sin_cache.scalar_type() == at::kFloat) { + launch(cos_sin_cache.data_ptr()); } else { - rotary_embedding_kernel<<>>( - positions.data_ptr(), - query.data_ptr(), - key.has_value() ? key->data_ptr() : nullptr, - cos_sin_cache.data_ptr(), - rot_dim, - query_stride, - key_stride, - head_stride, - num_heads, - num_kv_heads, - head_size); + launch(cos_sin_cache.data_ptr()); } }); } diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py index d35c7d9b2e..bc68dccf70 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py @@ -2,35 +2,12 @@ from typing import Literal, get_args -from sglang.multimodal_gen.runtime.layers.quantization.auto_round import ( - AutoRoundConfig, -) -from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( - BitsAndBytesConfig, -) from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( - ConvRotInt8Config, -) -from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config from sglang.multimodal_gen.runtime.layers.quantization.method_names import ( canonical_quantization_method, ) -from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import ( - ModelOptFp8Config as ModelOptFp8DiffusionConfig, -) -from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( - ModelOptFp4Config, - ModelOptFp8Config, -) -from sglang.multimodal_gen.runtime.layers.quantization.modelslim import ModelSlimConfig -from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config -from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import ( - NPUMXFP4Config, -) -from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config QuantizationMethods = Literal[ "auto-round", @@ -51,19 +28,7 @@ QuantizationMethods = Literal[ QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods)) # The customized quantization methods which will be added to this dict. -_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = { - "auto-round": AutoRoundConfig, - "modelopt": ModelOptFp8DiffusionConfig, - "modelopt_fp8": ModelOptFp8Config, - "modelopt_fp4": ModelOptFp4Config, - "bitsandbytes": BitsAndBytesConfig, - "modelslim": ModelSlimConfig, - "fp8": Fp8Config, - "mxfp4": Mxfp4Config, - "mxfp8": MXFP8Config, - "mxfp4_npu": NPUMXFP4Config, - "convrot_int8": ConvRotInt8Config, -} +_CUSTOMIZED_METHOD_TO_QUANT_CONFIG: dict[str, type[QuantizationConfig]] = {} def register_quantization_config(quantization: str): @@ -99,7 +64,46 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: if quantization not in QUANTIZATION_METHODS: raise ValueError(f"Invalid quantization method: {quantization}") - method_to_config: dict[str, type[QuantizationConfig]] = {} + # Quantized linear methods import LinearBase; load them after module initialization. + from sglang.multimodal_gen.runtime.layers.quantization.auto_round import ( + AutoRoundConfig, + ) + from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( + BitsAndBytesConfig, + ) + from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, + ) + from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config + from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import ( + ModelOptFp8Config as ModelOptFp8DiffusionConfig, + ) + from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( + ModelOptFp4Config, + ModelOptFp8Config, + ) + from sglang.multimodal_gen.runtime.layers.quantization.modelslim import ( + ModelSlimConfig, + ) + from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config + from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import ( + NPUMXFP4Config, + ) + from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config + + method_to_config: dict[str, type[QuantizationConfig]] = { + "auto-round": AutoRoundConfig, + "modelopt": ModelOptFp8DiffusionConfig, + "modelopt_fp8": ModelOptFp8Config, + "modelopt_fp4": ModelOptFp4Config, + "bitsandbytes": BitsAndBytesConfig, + "modelslim": ModelSlimConfig, + "fp8": Fp8Config, + "mxfp4": Mxfp4Config, + "mxfp8": MXFP8Config, + "mxfp4_npu": NPUMXFP4Config, + "convrot_int8": ConvRotInt8Config, + } # Update the `method_to_config` with customized quantization methods. method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index a8b6c0628c..690645e007 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -679,13 +679,14 @@ class MiniMaxH3DenoisingStage(DenoisingStage): if not ( current_platform.is_cuda() + or current_platform.is_hip() or current_platform.is_cpu() or current_platform.is_mps() or current_platform.is_npu() or current_platform.is_xpu() ): raise RuntimeError( - "MiniMax H3 full-loop denoise requires CPU, CUDA, MPS, XPU, or Ascend NPU" + "MiniMax H3 full-loop denoise requires CPU, CUDA, ROCm, MPS, XPU, or Ascend NPU" ) device = current_platform.get_local_torch_device() diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_import_isolation.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_import_isolation.py index 2d7f7856eb..06448370c1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_import_isolation.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_import_isolation.py @@ -5,8 +5,37 @@ import subprocess import sys import unittest +from sglang.test.test_utils import CustomTestCase + + +class TestDiffusionImportIsolation(CustomTestCase): + def test_linear_and_model_loader_import_independently(self): + script = """ +import importlib +import sys +importlib.import_module(sys.argv[1]) +from sglang.multimodal_gen.runtime.layers.quantization import get_quantization_config +from sglang.multimodal_gen.runtime.layers.quantization.auto_round import AutoRoundConfig +assert get_quantization_config("auto-round") is AutoRoundConfig +""" + for module in ( + "sglang.multimodal_gen.runtime.layers.linear", + "sglang.multimodal_gen.runtime.utils.hf_diffusers_utils", + ): + with self.subTest(module=module): + completed = subprocess.run( + [sys.executable, "-c", script, module], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) -class TestDiffusionImportIsolation(unittest.TestCase): def test_disabled_backend_does_not_import_mlx(self): """Diffusion modules must remain usable without the optional MLX path.""" script = """ diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index 87ef531261..26fa9bf579 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -25,6 +25,12 @@ done OPTIONAL_DEPS="${1:-}" +if [[ ",${OPTIONAL_DEPS}," == *,diffusion,* ]]; then + # MiniMax H3 uses ffprobe for input admission and output audio validation. + docker exec ci_sglang apt-get update + docker exec -e DEBIAN_FRONTEND=noninteractive ci_sglang apt-get install -y --no-install-recommends ffmpeg +fi + # Build python extras EXTRAS="dev_hip,tracing" if [ -n "$OPTIONAL_DEPS" ]; then diff --git a/test/registered/rotary/test_rope_rocm.py b/test/registered/rotary/test_rope_rocm.py index 3a982051cc..bf7068c55b 100644 --- a/test/registered/rotary/test_rope_rocm.py +++ b/test/registered/rotary/test_rope_rocm.py @@ -7,7 +7,7 @@ from sglang.srt.utils import get_bool_env_var, is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase -register_amd_ci(est_time=3, suite="stage-b-test-1-gpu-small-amd") +register_amd_ci(est_time=3, suite="stage-a-test-1-gpu-small-amd") torch.manual_seed(0) @@ -25,9 +25,9 @@ _CASES = [ ] -@unittest.skipIf(_use_aiter, reason="SGLANG_USE_AITER=1 will not use vllm path.") +@unittest.skipUnless(_is_hip, "Requires AMD GPU") class TestRotaryEmbeddingNative(CustomTestCase): - # Compare RotaryEmbedding.forward_hip() to forward_native(). + # SGLang's fallback remains in use by Llama when AITER is enabled. def _run_case( self, head_size: int, @@ -58,14 +58,35 @@ class TestRotaryEmbeddingNative(CustomTestCase): ) q_ref, k_ref = rope_ref.forward_native(pos_ids, query.clone(), key.clone()) - q_hip, k_hip = rope_hip.forward_hip(pos_ids, query.clone(), key.clone()) + q_hip, k_hip = rope_hip(pos_ids, query.clone(), key.clone()) + self.assertEqual(rope_hip.cos_sin_cache.dtype, torch.float32) + + # FP32 cache storage must preserve the existing query-dtype arithmetic. + from sgl_kernel import rotary_embedding + + q_same, k_same = query.clone(), key.clone() + rotary_embedding( + pos_ids, + q_same, + k_same, + head_size, + rope_hip.cos_sin_cache.to(dtype), + is_neox, + ) + torch.testing.assert_close(q_hip, q_same, atol=0, rtol=0) + torch.testing.assert_close(k_hip, k_same, atol=0, rtol=0) torch.testing.assert_close(q_ref, q_hip, atol=1e-2, rtol=1e-2) torch.testing.assert_close(k_ref, k_hip, atol=1e-2, rtol=1e-2) def test_all_cases(self) -> None: """Drive over the full parameter matrix using subTest().""" - for case in _CASES: + cases = _CASES + [ + (128, 64, 32, 10000, True, torch.float16, "cuda", 1, 32, 4, 2), + (128, 64, 32, 10000, False, torch.float16, "cuda", 1, 32, 4, 2), + (128, 64, 32, 10000, True, torch.float32, "cuda", 1, 32, 4, 2), + ] + for case in cases: with self.subTest(case=case): self._run_case(*case)