[AMD] Fix RoPE cache dtype and diffusion CI failures (#42494)

This commit is contained in:
Xiaoyu Zhang
2026-10-05 00:36:26 +08:00
committed by GitHub
parent 0343bc1483
commit 7a719e9a65
7 changed files with 171 additions and 92 deletions
@@ -5,16 +5,19 @@
#include <c10/cuda/CUDAGuard.h>
#include <torch/all.h>
#include <type_traits>
#include "utils.h"
template <typename scalar_t, bool IS_NEOX>
template <typename scalar_t, typename cache_t, bool IS_NEOX>
inline __device__ void apply_token_rotary_embedding(
scalar_t* __restrict__ arr,
const scalar_t* __restrict__ cos_ptr,
const scalar_t* __restrict__ sin_ptr,
const cache_t* __restrict__ cos_ptr,
const cache_t* __restrict__ sin_ptr,
int rot_offset,
int embed_dim) {
int x_index, y_index;
// Round cache values to the query dtype, as in the same-dtype cache path.
scalar_t cos, sin;
if (IS_NEOX) {
// GPT-NeoX style rotary embedding.
@@ -36,7 +39,7 @@ inline __device__ void apply_token_rotary_embedding(
arr[y_index] = y * cos + x * sin;
}
template <typename scalar_t, bool IS_NEOX>
template <typename scalar_t, typename cache_t, bool IS_NEOX>
inline __device__ void apply_rotary_embedding(
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
// head_size] or [num_tokens, num_heads,
@@ -45,7 +48,7 @@ inline __device__ void apply_rotary_embedding(
// [batch_size, seq_len, num_kv_heads,
// head_size] or [num_tokens, num_kv_heads,
// head_size]
const scalar_t* cache_ptr,
const cache_t* cache_ptr,
const int head_size,
const int num_heads,
const int num_kv_heads,
@@ -55,15 +58,16 @@ inline __device__ void apply_rotary_embedding(
const int64_t key_stride,
const int64_t head_stride) {
const int embed_dim = rot_dim / 2;
const scalar_t* cos_ptr = cache_ptr;
const scalar_t* sin_ptr = cache_ptr + embed_dim;
const cache_t* cos_ptr = cache_ptr;
const cache_t* sin_ptr = cache_ptr + embed_dim;
const int nq = num_heads * embed_dim;
for (int i = threadIdx.x; i < nq; i += blockDim.x) {
const int head_idx = i / embed_dim;
const int64_t token_head = token_idx * query_stride + head_idx * head_stride;
const int rot_offset = i % embed_dim;
apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
apply_token_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
}
if (key != nullptr) {
@@ -72,24 +76,25 @@ inline __device__ void apply_rotary_embedding(
const int head_idx = i / embed_dim;
const int64_t token_head = token_idx * key_stride + head_idx * head_stride;
const int rot_offset = i % embed_dim;
apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
apply_token_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
}
}
}
template <typename scalar_t, bool IS_NEOX>
template <typename scalar_t, typename cache_t, bool IS_NEOX>
__global__ void rotary_embedding_kernel(
const int64_t* __restrict__ positions, // [batch_size, seq_len] or
// [num_tokens]
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
// head_size] or [num_tokens, num_heads,
// head_size]
scalar_t* __restrict__ key, // nullptr or
// [batch_size, seq_len, num_kv_heads,
// head_size] or [num_tokens, num_kv_heads,
// head_size]
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
// 2]
const int64_t* __restrict__ positions, // [batch_size, seq_len] or
// [num_tokens]
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
// head_size] or [num_tokens, num_heads,
// head_size]
scalar_t* __restrict__ key, // nullptr or
// [batch_size, seq_len, num_kv_heads,
// head_size] or [num_tokens, num_kv_heads,
// head_size]
const cache_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
// 2]
const int rot_dim,
const int64_t query_stride,
const int64_t key_stride,
@@ -100,9 +105,9 @@ __global__ void rotary_embedding_kernel(
// Each thread block is responsible for one token.
const int token_idx = blockIdx.x;
int64_t pos = positions[token_idx];
const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
const cache_t* cache_ptr = cos_sin_cache + pos * rot_dim;
apply_rotary_embedding<scalar_t, IS_NEOX>(
apply_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
query,
key,
cache_ptr,
@@ -177,32 +182,40 @@ void rotary_embedding(
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
DISPATCH_FLOAT_TYPES(query.scalar_type(), "rotary_embedding", [&] {
if (is_neox) {
rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
positions.data_ptr<int64_t>(),
query.data_ptr<scalar_t>(),
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
cos_sin_cache.data_ptr<scalar_t>(),
rot_dim,
query_stride,
key_stride,
head_stride,
num_heads,
num_kv_heads,
head_size);
auto launch = [&](auto* cache_ptr) {
using cache_t = std::remove_pointer_t<decltype(cache_ptr)>;
if (is_neox) {
rotary_embedding_kernel<scalar_t, cache_t, true><<<grid, block, 0, stream>>>(
positions.data_ptr<int64_t>(),
query.data_ptr<scalar_t>(),
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
cache_ptr,
rot_dim,
query_stride,
key_stride,
head_stride,
num_heads,
num_kv_heads,
head_size);
} else {
rotary_embedding_kernel<scalar_t, cache_t, false><<<grid, block, 0, stream>>>(
positions.data_ptr<int64_t>(),
query.data_ptr<scalar_t>(),
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
cache_ptr,
rot_dim,
query_stride,
key_stride,
head_stride,
num_heads,
num_kv_heads,
head_size);
}
};
if (cos_sin_cache.scalar_type() == at::kFloat) {
launch(cos_sin_cache.data_ptr<float>());
} else {
rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
positions.data_ptr<int64_t>(),
query.data_ptr<scalar_t>(),
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
cos_sin_cache.data_ptr<scalar_t>(),
rot_dim,
query_stride,
key_stride,
head_stride,
num_heads,
num_kv_heads,
head_size);
launch(cos_sin_cache.data_ptr<scalar_t>());
}
});
}
@@ -2,35 +2,12 @@
from typing import Literal, get_args
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import (
AutoRoundConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
BitsAndBytesConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
canonical_quantization_method,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp4Config,
ModelOptFp8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelslim import ModelSlimConfig
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import (
NPUMXFP4Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config
QuantizationMethods = Literal[
"auto-round",
@@ -51,19 +28,7 @@ QuantizationMethods = Literal[
QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
# The customized quantization methods which will be added to this dict.
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {
"auto-round": AutoRoundConfig,
"modelopt": ModelOptFp8DiffusionConfig,
"modelopt_fp8": ModelOptFp8Config,
"modelopt_fp4": ModelOptFp4Config,
"bitsandbytes": BitsAndBytesConfig,
"modelslim": ModelSlimConfig,
"fp8": Fp8Config,
"mxfp4": Mxfp4Config,
"mxfp8": MXFP8Config,
"mxfp4_npu": NPUMXFP4Config,
"convrot_int8": ConvRotInt8Config,
}
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG: dict[str, type[QuantizationConfig]] = {}
def register_quantization_config(quantization: str):
@@ -99,7 +64,46 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
if quantization not in QUANTIZATION_METHODS:
raise ValueError(f"Invalid quantization method: {quantization}")
method_to_config: dict[str, type[QuantizationConfig]] = {}
# Quantized linear methods import LinearBase; load them after module initialization.
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import (
AutoRoundConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
BitsAndBytesConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp4Config,
ModelOptFp8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelslim import (
ModelSlimConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import (
NPUMXFP4Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config
method_to_config: dict[str, type[QuantizationConfig]] = {
"auto-round": AutoRoundConfig,
"modelopt": ModelOptFp8DiffusionConfig,
"modelopt_fp8": ModelOptFp8Config,
"modelopt_fp4": ModelOptFp4Config,
"bitsandbytes": BitsAndBytesConfig,
"modelslim": ModelSlimConfig,
"fp8": Fp8Config,
"mxfp4": Mxfp4Config,
"mxfp8": MXFP8Config,
"mxfp4_npu": NPUMXFP4Config,
"convrot_int8": ConvRotInt8Config,
}
# Update the `method_to_config` with customized quantization methods.
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
@@ -679,13 +679,14 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
if not (
current_platform.is_cuda()
or current_platform.is_hip()
or current_platform.is_cpu()
or current_platform.is_mps()
or current_platform.is_npu()
or current_platform.is_xpu()
):
raise RuntimeError(
"MiniMax H3 full-loop denoise requires CPU, CUDA, MPS, XPU, or Ascend NPU"
"MiniMax H3 full-loop denoise requires CPU, CUDA, ROCm, MPS, XPU, or Ascend NPU"
)
device = current_platform.get_local_torch_device()
@@ -5,8 +5,37 @@ import subprocess
import sys
import unittest
from sglang.test.test_utils import CustomTestCase
class TestDiffusionImportIsolation(CustomTestCase):
def test_linear_and_model_loader_import_independently(self):
script = """
import importlib
import sys
importlib.import_module(sys.argv[1])
from sglang.multimodal_gen.runtime.layers.quantization import get_quantization_config
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import AutoRoundConfig
assert get_quantization_config("auto-round") is AutoRoundConfig
"""
for module in (
"sglang.multimodal_gen.runtime.layers.linear",
"sglang.multimodal_gen.runtime.utils.hf_diffusers_utils",
):
with self.subTest(module=module):
completed = subprocess.run(
[sys.executable, "-c", script, module],
capture_output=True,
text=True,
timeout=30,
check=False,
)
self.assertEqual(
completed.returncode,
0,
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
)
class TestDiffusionImportIsolation(unittest.TestCase):
def test_disabled_backend_does_not_import_mlx(self):
"""Diffusion modules must remain usable without the optional MLX path."""
script = """