[MUSA][18/N] Add MUSA-optimized kernel implementations for hot ops (#23255)

Signed-off-by: Joey-gvwal <joey_gvwal@yeah.net>
Co-authored-by: R0CKSTAR <yeahdongcn@gmail.com>

sgl-kernel/include/musa/dispatch_utils.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2020-2026, Moore Threads Technology Co., Ltd("Moore Threads").
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Adapted from
+ * https://github.com/pytorch/pytorch/blob/v2.0.1/aten/src/ATen/Dispatch.h
+ */
+#pragma once
+
+#include <torch/all.h>
+
+#define MUSA_LDG(arg) __ldg(arg)
+
+#define MUSA_DISPATCH_CASE_FLOATING_TYPES(...)         \
+  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
+  AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)  \
+  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
+
+#define MUSA_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
+  AT_DISPATCH_SWITCH(TYPE, NAME, MUSA_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))

sgl-kernel/include/musa/integer_subbyte.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2020-2026, Moore Threads Technology Co., Ltd("Moore Threads").
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <type_traits>
+
+namespace musa::dnn {
+
+// cutlass integer_subbyte class
+template <int Bits, bool Signed = true>
+struct integer_subbyte {
+  using Storage = uint8_t;
+
+  static_assert(Bits <= 8 * sizeof(Storage), "Require a subbyte of bits in integer_subbyte");
+
+  using xint_t = typename std::conditional<Signed, int, unsigned>::type;
+
+  static constexpr Storage bits_mask_ = Storage((1 << Bits) - 1);
+
+  static constexpr Storage sign_mask_ = Storage((Signed ? 1 : 0) << (Bits - 1));
+
+  Storage storage;
+
+  __host__ __device__ constexpr integer_subbyte() {}
+
+  __host__ __device__ constexpr integer_subbyte(int value)
+      : storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
+
+  __host__ __device__ constexpr integer_subbyte(unsigned value)
+      : storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
+};
+
+}  // namespace musa::dnn

sgl-kernel/include/sgl_kernel_musa_ops.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020-2026, Moore Threads Technology Co., Ltd("Moore Threads").
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <ATen/ATen.h>
+#include <ATen/Tensor.h>
+#include <torch/torch.h>
+
+#include <optional>
+
+void batched_rotary_embedding_contiguous(
+    torch::Tensor& positions,
+    torch::Tensor& query,
+    torch::Tensor& key,
+    int64_t head_size,
+    torch::Tensor& cos_sin_cache,
+    bool is_neox,
+    int64_t rot_dim,
+    torch::Tensor& cos_sin_cache_offsets);
+
+void rotary_embedding_contiguous(
+    torch::Tensor& positions,
+    torch::Tensor& query,
+    torch::Tensor& key,
+    int64_t head_size,
+    torch::Tensor& cos_sin_cache,
+    bool is_neox);
+
+void fused_moe_gemv(
+    torch::Tensor& A,
+    torch::Tensor& B,
+    torch::Tensor& C,
+    const c10::optional<torch::Tensor>& A_scale,
+    const c10::optional<torch::Tensor>& B_scale,
+    torch::Tensor& topk_weights,
+    torch::Tensor& topk_ids,
+    bool mul_routed_weight,
+    int64_t topk,
+    bool use_int4_w4a16,
+    bool use_swigelu);
+
+void musa_fused_gemv(
+    torch::Tensor& A,
+    torch::Tensor& B,
+    torch::Tensor& C,
+    const c10::optional<torch::Tensor>& A_scale,
+    const c10::optional<torch::Tensor>& B_scale,
+    bool use_int4_w4a16,
+    bool use_swigelu,
+    bool use_rms_norm,
+    const c10::optional<torch::Tensor>& gamma,
+    double eps);
+
+void fused_mul_add(torch::Tensor& output, torch::Tensor& self, torch::Tensor& bias, double scale);
+
+void musa_top_k_top_p_sampling_from_probs(
+    at::Tensor probs,
+    at::Tensor output,
+    std::optional<at::Tensor> maybe_indices,
+    std::optional<at::Tensor> maybe_top_k_arr,
+    double top_k_val,
+    std::optional<at::Tensor> maybe_top_p_arr,
+    double top_p_val,
+    bool deterministic,
+    std::optional<at::Generator> gen);

sgl-kernel/include/sgl_kernel_ops.h

@@ -27,6 +27,10 @@ limitations under the License.
 
 #include "scalar_type.hpp"
 
+#ifdef USE_MUSA
+#include "sgl_kernel_musa_ops.h"
+#endif
+
 #define _CONCAT(A, B) A##B
 #define CONCAT(A, B) _CONCAT(A, B)