[AMD] Fix RoPE cache dtype and diffusion CI failures (#42494)
This commit is contained in:
@@ -5,16 +5,19 @@
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
template <typename scalar_t, bool IS_NEOX>
|
||||
template <typename scalar_t, typename cache_t, bool IS_NEOX>
|
||||
inline __device__ void apply_token_rotary_embedding(
|
||||
scalar_t* __restrict__ arr,
|
||||
const scalar_t* __restrict__ cos_ptr,
|
||||
const scalar_t* __restrict__ sin_ptr,
|
||||
const cache_t* __restrict__ cos_ptr,
|
||||
const cache_t* __restrict__ sin_ptr,
|
||||
int rot_offset,
|
||||
int embed_dim) {
|
||||
int x_index, y_index;
|
||||
// Round cache values to the query dtype, as in the same-dtype cache path.
|
||||
scalar_t cos, sin;
|
||||
if (IS_NEOX) {
|
||||
// GPT-NeoX style rotary embedding.
|
||||
@@ -36,7 +39,7 @@ inline __device__ void apply_token_rotary_embedding(
|
||||
arr[y_index] = y * cos + x * sin;
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool IS_NEOX>
|
||||
template <typename scalar_t, typename cache_t, bool IS_NEOX>
|
||||
inline __device__ void apply_rotary_embedding(
|
||||
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
|
||||
// head_size] or [num_tokens, num_heads,
|
||||
@@ -45,7 +48,7 @@ inline __device__ void apply_rotary_embedding(
|
||||
// [batch_size, seq_len, num_kv_heads,
|
||||
// head_size] or [num_tokens, num_kv_heads,
|
||||
// head_size]
|
||||
const scalar_t* cache_ptr,
|
||||
const cache_t* cache_ptr,
|
||||
const int head_size,
|
||||
const int num_heads,
|
||||
const int num_kv_heads,
|
||||
@@ -55,15 +58,16 @@ inline __device__ void apply_rotary_embedding(
|
||||
const int64_t key_stride,
|
||||
const int64_t head_stride) {
|
||||
const int embed_dim = rot_dim / 2;
|
||||
const scalar_t* cos_ptr = cache_ptr;
|
||||
const scalar_t* sin_ptr = cache_ptr + embed_dim;
|
||||
const cache_t* cos_ptr = cache_ptr;
|
||||
const cache_t* sin_ptr = cache_ptr + embed_dim;
|
||||
|
||||
const int nq = num_heads * embed_dim;
|
||||
for (int i = threadIdx.x; i < nq; i += blockDim.x) {
|
||||
const int head_idx = i / embed_dim;
|
||||
const int64_t token_head = token_idx * query_stride + head_idx * head_stride;
|
||||
const int rot_offset = i % embed_dim;
|
||||
apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
|
||||
apply_token_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
|
||||
query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
|
||||
}
|
||||
|
||||
if (key != nullptr) {
|
||||
@@ -72,24 +76,25 @@ inline __device__ void apply_rotary_embedding(
|
||||
const int head_idx = i / embed_dim;
|
||||
const int64_t token_head = token_idx * key_stride + head_idx * head_stride;
|
||||
const int rot_offset = i % embed_dim;
|
||||
apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
|
||||
apply_token_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
|
||||
key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool IS_NEOX>
|
||||
template <typename scalar_t, typename cache_t, bool IS_NEOX>
|
||||
__global__ void rotary_embedding_kernel(
|
||||
const int64_t* __restrict__ positions, // [batch_size, seq_len] or
|
||||
// [num_tokens]
|
||||
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
|
||||
// head_size] or [num_tokens, num_heads,
|
||||
// head_size]
|
||||
scalar_t* __restrict__ key, // nullptr or
|
||||
// [batch_size, seq_len, num_kv_heads,
|
||||
// head_size] or [num_tokens, num_kv_heads,
|
||||
// head_size]
|
||||
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
|
||||
// 2]
|
||||
const int64_t* __restrict__ positions, // [batch_size, seq_len] or
|
||||
// [num_tokens]
|
||||
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
|
||||
// head_size] or [num_tokens, num_heads,
|
||||
// head_size]
|
||||
scalar_t* __restrict__ key, // nullptr or
|
||||
// [batch_size, seq_len, num_kv_heads,
|
||||
// head_size] or [num_tokens, num_kv_heads,
|
||||
// head_size]
|
||||
const cache_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
|
||||
// 2]
|
||||
const int rot_dim,
|
||||
const int64_t query_stride,
|
||||
const int64_t key_stride,
|
||||
@@ -100,9 +105,9 @@ __global__ void rotary_embedding_kernel(
|
||||
// Each thread block is responsible for one token.
|
||||
const int token_idx = blockIdx.x;
|
||||
int64_t pos = positions[token_idx];
|
||||
const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
|
||||
const cache_t* cache_ptr = cos_sin_cache + pos * rot_dim;
|
||||
|
||||
apply_rotary_embedding<scalar_t, IS_NEOX>(
|
||||
apply_rotary_embedding<scalar_t, cache_t, IS_NEOX>(
|
||||
query,
|
||||
key,
|
||||
cache_ptr,
|
||||
@@ -177,32 +182,40 @@ void rotary_embedding(
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
DISPATCH_FLOAT_TYPES(query.scalar_type(), "rotary_embedding", [&] {
|
||||
if (is_neox) {
|
||||
rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
|
||||
positions.data_ptr<int64_t>(),
|
||||
query.data_ptr<scalar_t>(),
|
||||
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
|
||||
cos_sin_cache.data_ptr<scalar_t>(),
|
||||
rot_dim,
|
||||
query_stride,
|
||||
key_stride,
|
||||
head_stride,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_size);
|
||||
auto launch = [&](auto* cache_ptr) {
|
||||
using cache_t = std::remove_pointer_t<decltype(cache_ptr)>;
|
||||
if (is_neox) {
|
||||
rotary_embedding_kernel<scalar_t, cache_t, true><<<grid, block, 0, stream>>>(
|
||||
positions.data_ptr<int64_t>(),
|
||||
query.data_ptr<scalar_t>(),
|
||||
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
|
||||
cache_ptr,
|
||||
rot_dim,
|
||||
query_stride,
|
||||
key_stride,
|
||||
head_stride,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_size);
|
||||
} else {
|
||||
rotary_embedding_kernel<scalar_t, cache_t, false><<<grid, block, 0, stream>>>(
|
||||
positions.data_ptr<int64_t>(),
|
||||
query.data_ptr<scalar_t>(),
|
||||
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
|
||||
cache_ptr,
|
||||
rot_dim,
|
||||
query_stride,
|
||||
key_stride,
|
||||
head_stride,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_size);
|
||||
}
|
||||
};
|
||||
if (cos_sin_cache.scalar_type() == at::kFloat) {
|
||||
launch(cos_sin_cache.data_ptr<float>());
|
||||
} else {
|
||||
rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
|
||||
positions.data_ptr<int64_t>(),
|
||||
query.data_ptr<scalar_t>(),
|
||||
key.has_value() ? key->data_ptr<scalar_t>() : nullptr,
|
||||
cos_sin_cache.data_ptr<scalar_t>(),
|
||||
rot_dim,
|
||||
query_stride,
|
||||
key_stride,
|
||||
head_stride,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_size);
|
||||
launch(cos_sin_cache.data_ptr<scalar_t>());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,35 +2,12 @@
|
||||
|
||||
from typing import Literal, get_args
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import (
|
||||
AutoRoundConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
|
||||
BitsAndBytesConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
|
||||
canonical_quantization_method,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
|
||||
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4Config,
|
||||
ModelOptFp8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelslim import ModelSlimConfig
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import (
|
||||
NPUMXFP4Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config
|
||||
|
||||
QuantizationMethods = Literal[
|
||||
"auto-round",
|
||||
@@ -51,19 +28,7 @@ QuantizationMethods = Literal[
|
||||
QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
|
||||
|
||||
# The customized quantization methods which will be added to this dict.
|
||||
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {
|
||||
"auto-round": AutoRoundConfig,
|
||||
"modelopt": ModelOptFp8DiffusionConfig,
|
||||
"modelopt_fp8": ModelOptFp8Config,
|
||||
"modelopt_fp4": ModelOptFp4Config,
|
||||
"bitsandbytes": BitsAndBytesConfig,
|
||||
"modelslim": ModelSlimConfig,
|
||||
"fp8": Fp8Config,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"mxfp8": MXFP8Config,
|
||||
"mxfp4_npu": NPUMXFP4Config,
|
||||
"convrot_int8": ConvRotInt8Config,
|
||||
}
|
||||
_CUSTOMIZED_METHOD_TO_QUANT_CONFIG: dict[str, type[QuantizationConfig]] = {}
|
||||
|
||||
|
||||
def register_quantization_config(quantization: str):
|
||||
@@ -99,7 +64,46 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
if quantization not in QUANTIZATION_METHODS:
|
||||
raise ValueError(f"Invalid quantization method: {quantization}")
|
||||
|
||||
method_to_config: dict[str, type[QuantizationConfig]] = {}
|
||||
# Quantized linear methods import LinearBase; load them after module initialization.
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import (
|
||||
AutoRoundConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
|
||||
BitsAndBytesConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
|
||||
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4Config,
|
||||
ModelOptFp8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelslim import (
|
||||
ModelSlimConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4 import Mxfp4Config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp4_npu import (
|
||||
NPUMXFP4Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.mxfp8 import MXFP8Config
|
||||
|
||||
method_to_config: dict[str, type[QuantizationConfig]] = {
|
||||
"auto-round": AutoRoundConfig,
|
||||
"modelopt": ModelOptFp8DiffusionConfig,
|
||||
"modelopt_fp8": ModelOptFp8Config,
|
||||
"modelopt_fp4": ModelOptFp4Config,
|
||||
"bitsandbytes": BitsAndBytesConfig,
|
||||
"modelslim": ModelSlimConfig,
|
||||
"fp8": Fp8Config,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"mxfp8": MXFP8Config,
|
||||
"mxfp4_npu": NPUMXFP4Config,
|
||||
"convrot_int8": ConvRotInt8Config,
|
||||
}
|
||||
# Update the `method_to_config` with customized quantization methods.
|
||||
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
|
||||
|
||||
|
||||
+2
-1
@@ -679,13 +679,14 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
or current_platform.is_hip()
|
||||
or current_platform.is_cpu()
|
||||
or current_platform.is_mps()
|
||||
or current_platform.is_npu()
|
||||
or current_platform.is_xpu()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"MiniMax H3 full-loop denoise requires CPU, CUDA, MPS, XPU, or Ascend NPU"
|
||||
"MiniMax H3 full-loop denoise requires CPU, CUDA, ROCm, MPS, XPU, or Ascend NPU"
|
||||
)
|
||||
|
||||
device = current_platform.get_local_torch_device()
|
||||
|
||||
@@ -5,8 +5,37 @@ import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestDiffusionImportIsolation(CustomTestCase):
|
||||
def test_linear_and_model_loader_import_independently(self):
|
||||
script = """
|
||||
import importlib
|
||||
import sys
|
||||
importlib.import_module(sys.argv[1])
|
||||
from sglang.multimodal_gen.runtime.layers.quantization import get_quantization_config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.auto_round import AutoRoundConfig
|
||||
assert get_quantization_config("auto-round") is AutoRoundConfig
|
||||
"""
|
||||
for module in (
|
||||
"sglang.multimodal_gen.runtime.layers.linear",
|
||||
"sglang.multimodal_gen.runtime.utils.hf_diffusers_utils",
|
||||
):
|
||||
with self.subTest(module=module):
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", script, module],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode,
|
||||
0,
|
||||
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
||||
)
|
||||
|
||||
class TestDiffusionImportIsolation(unittest.TestCase):
|
||||
def test_disabled_backend_does_not_import_mlx(self):
|
||||
"""Diffusion modules must remain usable without the optional MLX path."""
|
||||
script = """
|
||||
|
||||
Reference in New Issue
Block a user