[Perf] moe: Preserve Marlin workspace and route-count budgets (#45462)
This commit is contained in:
@@ -34,7 +34,10 @@ template <
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float, // is zero point of float16 type?
|
||||
const bool kIsEP, // expert parallelism
|
||||
const bool kHasBias // has per-expert bias
|
||||
const bool kHasBias, // has per-expert bias
|
||||
const bool kExpertIdsAreSorted, // sorted block expert IDs
|
||||
const bool kZeroInactiveOutput, // zero rows in inactive routing blocks
|
||||
const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair
|
||||
>
|
||||
__global__ void Marlin(MARLIN_KERNEL_PARAMS);
|
||||
|
||||
|
||||
@@ -55,7 +55,10 @@ template <
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float, // is zero point of float16 type?
|
||||
const bool kIsEP, // expert parallelism
|
||||
const bool kHasBias // has per-expert bias
|
||||
const bool kHasBias, // has per-expert bias
|
||||
const bool kExpertIdsAreSorted, // sorted block expert IDs
|
||||
const bool kZeroInactiveOutput, // zero rows in inactive routing blocks
|
||||
const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair
|
||||
>
|
||||
__global__ void Marlin(
|
||||
const int4* __restrict__ A, // fp16 input matrix of shape mxk
|
||||
@@ -87,6 +90,27 @@ __global__ void Marlin(
|
||||
|
||||
#else
|
||||
|
||||
template <bool kExpertIdsAreSorted, int kMoeBlockSize>
|
||||
__device__ __forceinline__ void zero_inactive_output_rows(
|
||||
int4* __restrict__ output,
|
||||
const int32_t* __restrict__ sorted_token_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
int num_blocks,
|
||||
int num_routes,
|
||||
int row_vectors) {
|
||||
for (int block = blockIdx.x; block < num_blocks; block += gridDim.x) {
|
||||
if constexpr (!kExpertIdsAreSorted) {
|
||||
if (expert_ids[block] != -1) continue;
|
||||
}
|
||||
for (int offset = threadIdx.x; offset < kMoeBlockSize * row_vectors; offset += blockDim.x) {
|
||||
int route = sorted_token_ids[block * kMoeBlockSize + offset / row_vectors];
|
||||
if (route < num_routes) {
|
||||
output[static_cast<int64_t>(route) * row_vectors + offset % row_vectors] = {0, 0, 0, 0};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
|
||||
// output/accumulation.
|
||||
template <typename scalar_t>
|
||||
@@ -298,7 +322,10 @@ template <
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float, // is zero point of float16 type?
|
||||
const bool kIsEP, // expert parallelism
|
||||
const bool kHasBias // has per-expert bias
|
||||
const bool kHasBias, // has per-expert bias
|
||||
const bool kExpertIdsAreSorted, // sorted block expert IDs
|
||||
const bool kZeroInactiveOutput, // zero rows in inactive routing blocks
|
||||
const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair
|
||||
>
|
||||
__global__ void Marlin(
|
||||
const int4* __restrict__ A, // fp16 input matrix of shape mxk
|
||||
@@ -381,19 +408,71 @@ __global__ void Marlin(
|
||||
is_zp_float ? prob_n * prob_k / group_size / 8 : prob_n * prob_k / group_size / (pack_factor * 4);
|
||||
const int b_bias_expert_stride = prob_n / 8;
|
||||
|
||||
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
static_assert(!kHasInactivePrefixCount || (kExpertIdsAreSorted && !kIsEP));
|
||||
int num_tokens_past_padded;
|
||||
int inactive_prefix_blocks = 0;
|
||||
if constexpr (kHasInactivePrefixCount) {
|
||||
// One aligned transaction publishes both quantities without a dependent head-ID load.
|
||||
const int2 counts = __ldg(reinterpret_cast<const int2*>(num_tokens_past_padded_ptr));
|
||||
num_tokens_past_padded = counts.x;
|
||||
inactive_prefix_blocks = counts.y;
|
||||
} else {
|
||||
num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
}
|
||||
int parallel = num_tokens_past_padded / moe_block_size;
|
||||
// Every warp counts the same inactive blocks without a serial per-thread scan.
|
||||
int num_invalid_blocks = 0;
|
||||
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
|
||||
num_invalid_blocks += expert_ids_ptr[i] == -1;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
|
||||
}
|
||||
parallel -= num_invalid_blocks;
|
||||
if (parallel == 0) return;
|
||||
int num_invalid_blocks = 0;
|
||||
if constexpr (kHasInactivePrefixCount) {
|
||||
if (inactive_prefix_blocks > 0) {
|
||||
if constexpr (kZeroInactiveOutput) {
|
||||
zero_inactive_output_rows<true, moe_block_size>(
|
||||
C, sorted_token_ids_ptr, expert_ids_ptr, inactive_prefix_blocks, prob_m * top_k, prob_n / 8);
|
||||
}
|
||||
parallel -= inactive_prefix_blocks;
|
||||
if (parallel == 0) return;
|
||||
expert_ids_ptr += inactive_prefix_blocks;
|
||||
sorted_token_ids_ptr += inactive_prefix_blocks * moe_block_size;
|
||||
}
|
||||
} else if constexpr (kExpertIdsAreSorted) {
|
||||
if (expert_ids_ptr[0] < 0) {
|
||||
// Only the used prefix is sorted; allocated tail entries are undefined.
|
||||
int first = 0;
|
||||
int last = parallel;
|
||||
while (first < last) {
|
||||
int middle = first + (last - first) / 2;
|
||||
if (expert_ids_ptr[middle] < 0)
|
||||
first = middle + 1;
|
||||
else
|
||||
last = middle;
|
||||
}
|
||||
if constexpr (kZeroInactiveOutput) {
|
||||
// Preserve the original route table until its inactive destinations are written.
|
||||
zero_inactive_output_rows<true, moe_block_size>(
|
||||
C, sorted_token_ids_ptr, expert_ids_ptr, first, prob_m * top_k, prob_n / 8);
|
||||
}
|
||||
parallel -= first;
|
||||
if (parallel == 0) return;
|
||||
expert_ids_ptr += first;
|
||||
sorted_token_ids_ptr += first * moe_block_size;
|
||||
}
|
||||
} else {
|
||||
// Generic and EP callers may have scattered inactive blocks.
|
||||
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
|
||||
num_invalid_blocks += expert_ids_ptr[i] == -1;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
|
||||
}
|
||||
if constexpr (kZeroInactiveOutput) {
|
||||
if (num_invalid_blocks > 0) {
|
||||
zero_inactive_output_rows<false, moe_block_size>(
|
||||
C, sorted_token_ids_ptr, expert_ids_ptr, parallel, prob_m * top_k, prob_n / 8);
|
||||
}
|
||||
}
|
||||
parallel -= num_invalid_blocks;
|
||||
if (parallel == 0) return;
|
||||
}
|
||||
|
||||
int k_tiles = prob_k / 16 / thread_k_blocks;
|
||||
int n_tiles = prob_n / 16 / thread_n_blocks;
|
||||
@@ -521,22 +600,27 @@ __global__ void Marlin(
|
||||
if (par_id >= parallel) return;
|
||||
|
||||
old_expert_id = expert_id;
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
skip_count--;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
if constexpr (kExpertIdsAreSorted) {
|
||||
block_id = par_id;
|
||||
expert_id = expert_ids_ptr[block_id];
|
||||
} else {
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
skip_count--;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
expert_id = expert_ids_ptr[block_id];
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (w_type == host::kFE2M1f && s_type == host::kFE4M3fn) {
|
||||
|
||||
@@ -315,7 +315,10 @@ bool is_valid_config(
|
||||
GROUP_BLOCKS, \
|
||||
IS_ZP_FLOAT, \
|
||||
kIsEP, \
|
||||
kHasBias>; \
|
||||
kHasBias, \
|
||||
kExpertIdsAreSorted, \
|
||||
kZeroInactiveOutput, \
|
||||
kHasInactivePrefixCount>; \
|
||||
}
|
||||
|
||||
// COMMON: cases for (group_blocks in [-1, 2, 4, 8] and is_zp_float == false)
|
||||
@@ -440,7 +443,13 @@ bool is_valid_config(
|
||||
ACT_GET_IF_M234(W_TYPE, 16, 4, 256) \
|
||||
ACT_GET_IF_M234(W_TYPE, 8, 4, 128)
|
||||
|
||||
template <typename scalar_t, bool kIsEP, bool kHasBias>
|
||||
template <
|
||||
typename scalar_t,
|
||||
bool kIsEP,
|
||||
bool kHasBias,
|
||||
bool kExpertIdsAreSorted,
|
||||
bool kZeroInactiveOutput,
|
||||
bool kHasInactivePrefixCount>
|
||||
MarlinFuncPtr get_marlin_kernel(
|
||||
const host::ScalarType q_type,
|
||||
int thread_m_blocks,
|
||||
@@ -476,7 +485,13 @@ MarlinFuncPtr get_marlin_kernel(
|
||||
return kernel;
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool kIsEP, bool kHasBias>
|
||||
template <
|
||||
typename scalar_t,
|
||||
bool kIsEP,
|
||||
bool kHasBias,
|
||||
bool kExpertIdsAreSorted,
|
||||
bool kZeroInactiveOutput,
|
||||
bool kHasInactivePrefixCount>
|
||||
exec_config_t determine_exec_config(
|
||||
const host::ScalarType& q_type,
|
||||
int prob_m,
|
||||
@@ -549,17 +564,18 @@ exec_config_t determine_exec_config(
|
||||
group_blocks = group_size == -1 ? -1 : (group_size / 16);
|
||||
}
|
||||
|
||||
auto kernel = get_marlin_kernel<scalar_t, kIsEP, kHasBias>(
|
||||
q_type,
|
||||
thread_m_blocks,
|
||||
th_config.thread_n / 16,
|
||||
th_config.thread_k / 16,
|
||||
m_block_size_8,
|
||||
has_act_order,
|
||||
has_zp,
|
||||
group_blocks,
|
||||
th_config.num_threads,
|
||||
is_zp_float);
|
||||
auto kernel =
|
||||
get_marlin_kernel<scalar_t, kIsEP, kHasBias, kExpertIdsAreSorted, kZeroInactiveOutput, kHasInactivePrefixCount>(
|
||||
q_type,
|
||||
thread_m_blocks,
|
||||
th_config.thread_n / 16,
|
||||
th_config.thread_k / 16,
|
||||
m_block_size_8,
|
||||
has_act_order,
|
||||
has_zp,
|
||||
group_blocks,
|
||||
th_config.num_threads,
|
||||
is_zp_float);
|
||||
|
||||
if (kernel == MarlinDefault) continue;
|
||||
|
||||
@@ -587,7 +603,13 @@ exec_config_t determine_exec_config(
|
||||
return exec_cfg;
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool kIsEP, bool kHasBias>
|
||||
template <
|
||||
typename scalar_t,
|
||||
bool kIsEP,
|
||||
bool kHasBias,
|
||||
bool kExpertIdsAreSorted,
|
||||
bool kZeroInactiveOutput,
|
||||
bool kHasInactivePrefixCount>
|
||||
void marlin_mm(
|
||||
const void* A,
|
||||
const void* B,
|
||||
@@ -727,7 +749,13 @@ void marlin_mm(
|
||||
host::RuntimeCheck(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k);
|
||||
} else {
|
||||
// Auto config
|
||||
exec_cfg = determine_exec_config<scalar_t, kIsEP, kHasBias>(
|
||||
exec_cfg = determine_exec_config<
|
||||
scalar_t,
|
||||
kIsEP,
|
||||
kHasBias,
|
||||
kExpertIdsAreSorted,
|
||||
kZeroInactiveOutput,
|
||||
kHasInactivePrefixCount>(
|
||||
q_type,
|
||||
prob_m,
|
||||
prob_n,
|
||||
@@ -799,17 +827,18 @@ void marlin_mm(
|
||||
", max_shared_mem = ",
|
||||
max_shared_mem);
|
||||
|
||||
auto kernel = get_marlin_kernel<scalar_t, kIsEP, kHasBias>(
|
||||
q_type,
|
||||
thread_m_blocks,
|
||||
thread_n_blocks,
|
||||
thread_k_blocks,
|
||||
m_block_size_8,
|
||||
has_act_order,
|
||||
has_zp,
|
||||
group_blocks,
|
||||
num_threads,
|
||||
is_zp_float);
|
||||
auto kernel =
|
||||
get_marlin_kernel<scalar_t, kIsEP, kHasBias, kExpertIdsAreSorted, kZeroInactiveOutput, kHasInactivePrefixCount>(
|
||||
q_type,
|
||||
thread_m_blocks,
|
||||
thread_n_blocks,
|
||||
thread_k_blocks,
|
||||
m_block_size_8,
|
||||
has_act_order,
|
||||
has_zp,
|
||||
group_blocks,
|
||||
num_threads,
|
||||
is_zp_float);
|
||||
|
||||
if (kernel == MarlinDefault) {
|
||||
host::Panic(
|
||||
@@ -850,7 +879,13 @@ void marlin_mm(
|
||||
|
||||
} // namespace device::marlin_moe
|
||||
|
||||
template <typename scalar_t, bool kIsEP, bool kHasBias>
|
||||
template <
|
||||
typename scalar_t,
|
||||
bool kIsEP,
|
||||
bool kHasBias,
|
||||
bool kExpertIdsAreSorted,
|
||||
bool kZeroInactiveOutput,
|
||||
bool kHasInactivePrefixCount>
|
||||
void moe_wna16_marlin_gemm(
|
||||
tvm::ffi::TensorView a,
|
||||
tvm::ffi::TensorView c,
|
||||
@@ -887,6 +922,8 @@ void moe_wna16_marlin_gemm(
|
||||
bool is_zp_float) {
|
||||
using namespace host;
|
||||
|
||||
static_assert(!kIsEP || !kExpertIdsAreSorted, "Sorted block IDs are specialized only without EP");
|
||||
static_assert(!kHasInactivePrefixCount || (kExpertIdsAreSorted && !kIsEP));
|
||||
RuntimeCheck(is_ep == kIsEP, "is_ep does not match the compiled Marlin MoE specialization");
|
||||
RuntimeCheck(has_bias == kHasBias, "has_bias does not match the compiled Marlin MoE specialization");
|
||||
|
||||
@@ -930,6 +967,9 @@ void moe_wna16_marlin_gemm(
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({-1, -1}).with_dtype<scalar_t>().with_device(device).verify(a);
|
||||
if constexpr (kHasInactivePrefixCount) {
|
||||
TensorMatcher({2}).with_dtype<int32_t>().with_device(device).ensure_alignment(8).verify(num_tokens_post_padded);
|
||||
}
|
||||
|
||||
device.verify(b_q_weight.device());
|
||||
RuntimeCheck(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
|
||||
@@ -1087,45 +1127,46 @@ void moe_wna16_marlin_gemm(
|
||||
// Early return for zero-size M (moved after all validation)
|
||||
if (size_m == 0) return;
|
||||
|
||||
device::marlin_moe::marlin_mm<scalar_t, kIsEP, kHasBias>(
|
||||
a.data_ptr(),
|
||||
b_q_weight.data_ptr(),
|
||||
c.data_ptr(),
|
||||
c_tmp.data_ptr(),
|
||||
b_bias.data_ptr(),
|
||||
b_scales.data_ptr(),
|
||||
global_scale.data_ptr(),
|
||||
b_zeros.data_ptr(),
|
||||
g_idx.data_ptr(),
|
||||
perm.data_ptr(),
|
||||
a_tmp.data_ptr(),
|
||||
sorted_token_ids.data_ptr(),
|
||||
expert_ids.data_ptr(),
|
||||
num_tokens_post_padded.data_ptr(),
|
||||
topk_weights.data_ptr(),
|
||||
static_cast<int>(moe_block_size),
|
||||
static_cast<int>(top_k),
|
||||
mul_topk_weights,
|
||||
is_ep,
|
||||
static_cast<int>(size_m),
|
||||
static_cast<int>(size_n),
|
||||
static_cast<int>(size_k),
|
||||
workspace.data_ptr(),
|
||||
b_q_type,
|
||||
has_bias,
|
||||
has_act_order,
|
||||
is_k_full,
|
||||
has_zp,
|
||||
static_cast<int>(num_groups),
|
||||
static_cast<int>(group_size),
|
||||
dev,
|
||||
stream,
|
||||
thread_k,
|
||||
thread_n,
|
||||
sms,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
is_zp_float);
|
||||
device::marlin_moe::
|
||||
marlin_mm<scalar_t, kIsEP, kHasBias, kExpertIdsAreSorted, kZeroInactiveOutput, kHasInactivePrefixCount>(
|
||||
a.data_ptr(),
|
||||
b_q_weight.data_ptr(),
|
||||
c.data_ptr(),
|
||||
c_tmp.data_ptr(),
|
||||
b_bias.data_ptr(),
|
||||
b_scales.data_ptr(),
|
||||
global_scale.data_ptr(),
|
||||
b_zeros.data_ptr(),
|
||||
g_idx.data_ptr(),
|
||||
perm.data_ptr(),
|
||||
a_tmp.data_ptr(),
|
||||
sorted_token_ids.data_ptr(),
|
||||
expert_ids.data_ptr(),
|
||||
num_tokens_post_padded.data_ptr(),
|
||||
topk_weights.data_ptr(),
|
||||
static_cast<int>(moe_block_size),
|
||||
static_cast<int>(top_k),
|
||||
mul_topk_weights,
|
||||
is_ep,
|
||||
static_cast<int>(size_m),
|
||||
static_cast<int>(size_n),
|
||||
static_cast<int>(size_k),
|
||||
workspace.data_ptr(),
|
||||
b_q_type,
|
||||
has_bias,
|
||||
has_act_order,
|
||||
is_k_full,
|
||||
has_zp,
|
||||
static_cast<int>(num_groups),
|
||||
static_cast<int>(group_size),
|
||||
dev,
|
||||
stream,
|
||||
thread_k,
|
||||
thread_n,
|
||||
sms,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
is_zp_float);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -23,12 +23,12 @@ struct AlignSingleTokenParams {
|
||||
const int32_t* __restrict__ topk_ids; // [1, topk]
|
||||
int32_t* __restrict__ sorted_ids; // [topk * block_size]
|
||||
int32_t* __restrict__ expert_ids; // [topk]
|
||||
int32_t* __restrict__ num_post; // [1]
|
||||
int32_t* __restrict__ num_post; // [1] or [2]
|
||||
uint32_t topk;
|
||||
uint32_t block_size;
|
||||
};
|
||||
|
||||
template <bool kUsePDL>
|
||||
template <bool kUsePDL, bool kWriteInactivePrefixCount = false>
|
||||
__global__ void align_single_token_kernel(const AlignSingleTokenParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
const uint32_t lane = threadIdx.x; // one warp
|
||||
@@ -61,16 +61,21 @@ __global__ void align_single_token_kernel(const AlignSingleTokenParams __grid_co
|
||||
params.sorted_ids[p] = static_cast<int32_t>(topk);
|
||||
}
|
||||
}
|
||||
if (lane == 0) {
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
const uint32_t inactive = __ballot_sync(0xffffffff, lane < topk && my_id < 0);
|
||||
if (lane == 0) {
|
||||
*reinterpret_cast<int2*>(params.num_post) = make_int2(topk * bs, __popc(inactive));
|
||||
}
|
||||
} else if (lane == 0) {
|
||||
params.num_post[0] = static_cast<int32_t>(topk * bs);
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
template <bool kUsePDL>
|
||||
template <bool kUsePDL, bool kWriteInactivePrefixCount = false>
|
||||
struct AlignSingleTokenKernel {
|
||||
static constexpr auto kernel = align_single_token_kernel<kUsePDL>;
|
||||
static constexpr auto kernel = align_single_token_kernel<kUsePDL, kWriteInactivePrefixCount>;
|
||||
|
||||
static void
|
||||
run(const tvm::ffi::TensorView topk_ids,
|
||||
@@ -86,6 +91,9 @@ struct AlignSingleTokenKernel {
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({One_, K_}).with_dtype<int32_t>().with_device(device).verify(topk_ids);
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
TensorMatcher({2}).with_dtype<int32_t>().with_device(device).ensure_alignment(8).verify(num_post);
|
||||
}
|
||||
|
||||
const auto topk = static_cast<uint32_t>(K_.unwrap());
|
||||
RuntimeCheck(One_.unwrap() == 1, "moe_align_single_token requires M == 1");
|
||||
|
||||
@@ -23,6 +23,7 @@ limitations under the License.
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <type_traits>
|
||||
|
||||
#ifndef WARP_SIZE
|
||||
#define WARP_SIZE 32
|
||||
@@ -37,6 +38,14 @@ using Vec = int4;
|
||||
|
||||
namespace moe {
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ int32_t sentinel_expert_bucket(scalar_t expert_id, int32_t num_buckets) {
|
||||
static_assert(std::is_same_v<scalar_t, int32_t> || std::is_same_v<scalar_t, int64_t>);
|
||||
using unsigned_t = std::make_unsigned_t<scalar_t>;
|
||||
const unsigned_t shifted = static_cast<unsigned_t>(expert_id) + unsigned_t{1};
|
||||
return shifted < static_cast<unsigned_t>(num_buckets) ? static_cast<int32_t>(shifted) : 0;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void count_and_sort_expert_tokens_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
@@ -53,7 +62,24 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool kPadExpertIds>
|
||||
template <typename scalar_t>
|
||||
__global__ void count_and_sort_checked_expert_tokens_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
int32_t* __restrict__ cumsum_buffer,
|
||||
size_t numel,
|
||||
int32_t num_buckets) {
|
||||
const size_t tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const size_t stride = blockDim.x * gridDim.x;
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
const int32_t expert_id = sentinel_expert_bucket(topk_ids[i], num_buckets);
|
||||
const int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1);
|
||||
sorted_token_ids[rank_post_pad] = i;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool kWriteInactivePrefixCount, bool kMapInvalidExpertsToSentinel, bool kPadExpertIds>
|
||||
__global__ void moe_align_block_size_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
@@ -101,7 +127,12 @@ __global__ void moe_align_block_size_kernel(
|
||||
__syncthreads();
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int expert_id = topk_ids[i] + 1;
|
||||
int expert_id;
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
expert_id = sentinel_expert_bucket(topk_ids[i], num_experts);
|
||||
} else {
|
||||
expert_id = topk_ids[i] + 1;
|
||||
}
|
||||
atomicAdd(&shared_counts[expert_id], 1);
|
||||
}
|
||||
|
||||
@@ -163,7 +194,12 @@ __global__ void moe_align_block_size_kernel(
|
||||
|
||||
if (tid == 0) {
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
*reinterpret_cast<int2*>(total_tokens_post_pad) =
|
||||
make_int2(s_total_tokens_post_pad, CEILDIV(shared_counts[0], block_size));
|
||||
} else {
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -190,7 +226,12 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (tid == 0) {
|
||||
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
*reinterpret_cast<int2*>(total_tokens_post_pad) =
|
||||
make_int2(s_total_tokens_post_pad, CEILDIV(shared_counts[0], block_size));
|
||||
} else {
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -241,7 +282,12 @@ __global__ void moe_align_block_size_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int32_t fill_threads, bool kPadExpertIds>
|
||||
template <
|
||||
typename scalar_t,
|
||||
int32_t fill_threads,
|
||||
bool kWriteInactivePrefixCount,
|
||||
bool kMapInvalidExpertsToSentinel,
|
||||
bool kPadExpertIds>
|
||||
__global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
@@ -285,7 +331,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
}
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int32_t expert_id = topk_ids[i] + 1;
|
||||
int32_t expert_id;
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
expert_id = sentinel_expert_bucket(topk_ids[i], num_experts);
|
||||
} else {
|
||||
expert_id = topk_ids[i] + 1;
|
||||
}
|
||||
++tokens_cnts[(tid + 1) * num_experts + expert_id];
|
||||
}
|
||||
|
||||
@@ -305,7 +356,11 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
for (int i = 1; i <= num_experts; ++i) {
|
||||
cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[stride * num_experts + i - 1], block_size) * block_size;
|
||||
}
|
||||
*total_tokens_post_pad = static_cast<int32_t>(cumsum[num_experts]);
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
*reinterpret_cast<int2*>(total_tokens_post_pad) = make_int2(cumsum[num_experts], cumsum[1] / block_size);
|
||||
} else {
|
||||
*total_tokens_post_pad = static_cast<int32_t>(cumsum[num_experts]);
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
@@ -322,7 +377,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
}
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int32_t expert_id = topk_ids[i] + 1;
|
||||
int32_t expert_id;
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
expert_id = sentinel_expert_bucket(topk_ids[i], num_experts);
|
||||
} else {
|
||||
expert_id = topk_ids[i] + 1;
|
||||
}
|
||||
int32_t rank_post_pad = tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id];
|
||||
sorted_token_ids[rank_post_pad] = i;
|
||||
++tokens_cnts[tid * num_experts + expert_id];
|
||||
@@ -338,7 +398,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
// With 1024 threads and EXPERTS_PER_THREAD=4, covers at most 4096 expert
|
||||
// indices. Since num_experts includes the +1 offset bucket, this supports
|
||||
// up to 4095 real experts.
|
||||
template <typename scalar_t, int EXPERTS_PER_THREAD, bool kPadExpertIds>
|
||||
template <
|
||||
typename scalar_t,
|
||||
int EXPERTS_PER_THREAD,
|
||||
bool kWriteInactivePrefixCount,
|
||||
bool kMapInvalidExpertsToSentinel,
|
||||
bool kPadExpertIds>
|
||||
__global__ void moe_align_block_size_kernel_v2(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
@@ -388,9 +453,14 @@ __global__ void moe_align_block_size_kernel_v2(
|
||||
__syncthreads();
|
||||
|
||||
for (size_t i = tid; i < numel; i += blockDim.x) {
|
||||
int expert_id = topk_ids[i] + 1; // +1 offset convention
|
||||
if (expert_id < num_experts) {
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
const int32_t expert_id = sentinel_expert_bucket(topk_ids[i], num_experts);
|
||||
atomicAdd(&shared_counts[expert_id], 1);
|
||||
} else {
|
||||
int expert_id = topk_ids[i] + 1; // +1 offset convention
|
||||
if (expert_id < num_experts) {
|
||||
atomicAdd(&shared_counts[expert_id], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,7 +509,12 @@ __global__ void moe_align_block_size_kernel_v2(
|
||||
// Last thread writes total
|
||||
if (tid == blockDim.x - 1) {
|
||||
cumsum[num_experts] = thread_prefix + thread_sum;
|
||||
*total_tokens_post_pad = thread_prefix + thread_sum;
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
*reinterpret_cast<int2*>(total_tokens_post_pad) =
|
||||
make_int2(thread_prefix + thread_sum, CEILDIV(shared_counts[0], block_size));
|
||||
} else {
|
||||
*total_tokens_post_pad = thread_prefix + thread_sum;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
@@ -462,7 +537,11 @@ __global__ void moe_align_block_size_kernel_v2(
|
||||
|
||||
} // namespace moe
|
||||
|
||||
template <typename scalar_t, bool kPadExpertIds>
|
||||
template <
|
||||
typename scalar_t,
|
||||
bool kWriteInactivePrefixCount = false,
|
||||
bool kMapInvalidExpertsToSentinel = false,
|
||||
bool kPadExpertIds = false>
|
||||
struct MoeAlignBlockSizeKernel {
|
||||
static void
|
||||
run(tvm::ffi::TensorView topk_ids,
|
||||
@@ -476,6 +555,9 @@ struct MoeAlignBlockSizeKernel {
|
||||
using namespace host;
|
||||
|
||||
auto device = topk_ids.device();
|
||||
if constexpr (kWriteInactivePrefixCount) {
|
||||
TensorMatcher({2}).with_dtype<int32_t>().with_device(device).ensure_alignment(8).verify(num_tokens_post_pad);
|
||||
}
|
||||
const cudaStream_t stream = LaunchKernel::resolve_device(device);
|
||||
|
||||
int threads = 1024;
|
||||
@@ -488,6 +570,10 @@ struct MoeAlignBlockSizeKernel {
|
||||
// to 8, covering at most 8192 expert indices. This supports up to 8191 real
|
||||
// experts, sufficient for LoRA virtual experts (num_moe_experts * max_loras).
|
||||
RuntimeCheck(num_experts <= 8192, "moe_align_block_size: num_experts must be <= 8192, got ", num_experts);
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
RuntimeCheck(
|
||||
num_experts > 0 && block_size > 0, "moe_align_block_size: bucket count and block size must be positive");
|
||||
}
|
||||
|
||||
const scalar_t* topk_ids_ptr = static_cast<const scalar_t*>(topk_ids.data_ptr());
|
||||
int32_t* sorted_token_ids_ptr = static_cast<int32_t*>(sorted_token_ids.data_ptr());
|
||||
@@ -503,7 +589,12 @@ struct MoeAlignBlockSizeKernel {
|
||||
constexpr int32_t fill_threads = 256;
|
||||
const int32_t shared_mem_size = ((num_thread + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
|
||||
|
||||
auto kernel = moe::moe_align_block_size_small_batch_expert_kernel<scalar_t, fill_threads, kPadExpertIds>;
|
||||
auto kernel = moe::moe_align_block_size_small_batch_expert_kernel<
|
||||
scalar_t,
|
||||
fill_threads,
|
||||
kWriteInactivePrefixCount,
|
||||
kMapInvalidExpertsToSentinel,
|
||||
kPadExpertIds>;
|
||||
LaunchKernel(dim3(1), dim3(fill_threads + num_thread), stream, shared_mem_size)(
|
||||
kernel,
|
||||
topk_ids_ptr,
|
||||
@@ -520,7 +611,8 @@ struct MoeAlignBlockSizeKernel {
|
||||
const size_t scan_size = host::round_up_pow2(static_cast<uint32_t>(num_experts));
|
||||
const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t);
|
||||
|
||||
auto align_kernel = moe::moe_align_block_size_kernel<scalar_t, kPadExpertIds>;
|
||||
auto align_kernel = moe::
|
||||
moe_align_block_size_kernel<scalar_t, kWriteInactivePrefixCount, kMapInvalidExpertsToSentinel, kPadExpertIds>;
|
||||
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
|
||||
align_kernel,
|
||||
topk_ids_ptr,
|
||||
@@ -541,9 +633,22 @@ struct MoeAlignBlockSizeKernel {
|
||||
const int max_blocks = 65535;
|
||||
const int actual_blocks = std::min(num_blocks, max_blocks);
|
||||
|
||||
auto sort_kernel = moe::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel);
|
||||
if (actual_blocks > 0) {
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
auto sort_kernel = moe::count_and_sort_checked_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel,
|
||||
topk_ids_ptr,
|
||||
sorted_token_ids_ptr,
|
||||
cumsum_buffer_ptr,
|
||||
numel,
|
||||
static_cast<int32_t>(num_experts));
|
||||
} else {
|
||||
auto sort_kernel = moe::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// v2 path for >1024 experts: two-level warp scan with EXPERTS_PER_THREAD
|
||||
int64_t padded_num_experts = ((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
@@ -551,7 +656,12 @@ struct MoeAlignBlockSizeKernel {
|
||||
|
||||
auto launch_v2 = [&](auto ept_tag) {
|
||||
constexpr int EPT = decltype(ept_tag)::value;
|
||||
auto v2_kernel = moe::moe_align_block_size_kernel_v2<scalar_t, EPT, kPadExpertIds>;
|
||||
auto v2_kernel = moe::moe_align_block_size_kernel_v2<
|
||||
scalar_t,
|
||||
EPT,
|
||||
kWriteInactivePrefixCount,
|
||||
kMapInvalidExpertsToSentinel,
|
||||
kPadExpertIds>;
|
||||
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
|
||||
v2_kernel,
|
||||
topk_ids_ptr,
|
||||
@@ -581,9 +691,22 @@ struct MoeAlignBlockSizeKernel {
|
||||
const int max_blocks = 65535;
|
||||
const int actual_blocks = std::min(num_blocks, max_blocks);
|
||||
|
||||
auto sort_kernel = moe::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel);
|
||||
if (actual_blocks > 0) {
|
||||
if constexpr (kMapInvalidExpertsToSentinel) {
|
||||
auto sort_kernel = moe::count_and_sort_checked_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel,
|
||||
topk_ids_ptr,
|
||||
sorted_token_ids_ptr,
|
||||
cumsum_buffer_ptr,
|
||||
numel,
|
||||
static_cast<int32_t>(num_experts));
|
||||
} else {
|
||||
auto sort_kernel = moe::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -71,6 +71,9 @@ def topk_softmax_pack(
|
||||
pack id = -1 (the padded-region sentinel). ``topk_weights``/``topk_indices``
|
||||
are written exactly like the AOT kernel (indices NOT masked here — the
|
||||
regular python post-process handles them).
|
||||
|
||||
The real-row count may be a scalar or a one-element vector. A scalar is
|
||||
viewed as the vector expected by the native launcher without copying it.
|
||||
"""
|
||||
assert gating_output.dim() == 2
|
||||
num_experts = gating_output.shape[-1]
|
||||
@@ -80,6 +83,8 @@ def topk_softmax_pack(
|
||||
)
|
||||
if gating_output.shape[0] == 0:
|
||||
return
|
||||
if num_token_non_padded is not None and num_token_non_padded.dim() == 0:
|
||||
num_token_non_padded = num_token_non_padded.view(1)
|
||||
_jit_topk_softmax_pack_op(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
|
||||
@@ -307,6 +307,9 @@ def _align_block_size_jit(
|
||||
block_size: int,
|
||||
num_experts: int,
|
||||
scratch=None,
|
||||
*,
|
||||
return_inactive_prefix_count: bool = False,
|
||||
map_invalid_experts_to_sentinel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""CUDA JIT align_block_size for num_experts > 1024 (up to 8191).
|
||||
|
||||
@@ -316,6 +319,8 @@ def _align_block_size_jit(
|
||||
|
||||
``scratch(numel)`` returns the int32 scratch buffer (default: a fresh
|
||||
allocation); a caller under CUDA graphs passes a persistent buffer.
|
||||
IDs must be certified in [-1, num_experts) unless
|
||||
``map_invalid_experts_to_sentinel`` maps other signed IDs to bucket zero.
|
||||
"""
|
||||
assert num_experts <= 8191, (
|
||||
f"_align_block_size_jit supports at most 8191 experts "
|
||||
@@ -324,13 +329,21 @@ def _align_block_size_jit(
|
||||
|
||||
device = topk_ids.device
|
||||
flat_topk_ids = topk_ids.reshape(-1)
|
||||
if flat_topk_ids.dtype == torch.int64:
|
||||
if not map_invalid_experts_to_sentinel and flat_topk_ids.dtype == torch.int64:
|
||||
flat_topk_ids = flat_topk_ids.to(torch.int32)
|
||||
num_total_tokens = flat_topk_ids.numel()
|
||||
|
||||
if num_total_tokens == 0:
|
||||
empty = torch.empty(0, dtype=torch.int32, device=device)
|
||||
return empty, empty, torch.zeros(1, dtype=torch.int32, device=device)
|
||||
return (
|
||||
empty,
|
||||
empty,
|
||||
torch.zeros(
|
||||
2 if return_inactive_prefix_count else 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
|
||||
# JIT kernel uses +1 offset convention: -1 -> bucket 0 (sentinel),
|
||||
# expert i -> bucket i+1. So pass num_experts + 1 as the bucket count.
|
||||
@@ -370,7 +383,9 @@ def _align_block_size_jit(
|
||||
off += max_num_tokens_padded
|
||||
expert_ids = buf[off : off + max_num_m_blocks]
|
||||
off += max_num_m_blocks_padded
|
||||
num_tokens_post_padded = buf[off : off + 1]
|
||||
# The already padded count segment has room for the optional pair:
|
||||
# [total padded tokens, leading negative-expert blocks].
|
||||
num_tokens_post_padded = buf[off : off + (2 if return_inactive_prefix_count else 1)]
|
||||
off += num_post_pad_size
|
||||
cumsum_buffer = buf[off : off + jit_num_experts + 1]
|
||||
|
||||
@@ -383,6 +398,8 @@ def _align_block_size_jit(
|
||||
num_tokens_post_padded,
|
||||
cumsum_buffer,
|
||||
True, # pad_sorted_token_ids
|
||||
write_inactive_prefix_count=return_inactive_prefix_count,
|
||||
map_invalid_experts_to_sentinel=map_invalid_experts_to_sentinel,
|
||||
)
|
||||
|
||||
return sorted_token_ids, expert_ids, num_tokens_post_padded
|
||||
|
||||
@@ -11,8 +11,20 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_moe_align_module(dtype: torch.dtype, pad_expert_ids: bool) -> Module:
|
||||
args = make_cpp_args(dtype, pad_expert_ids)
|
||||
def _jit_moe_align_module(
|
||||
dtype: torch.dtype,
|
||||
write_inactive_prefix_count: bool = False,
|
||||
map_invalid_experts_to_sentinel: bool = False,
|
||||
pad_expert_ids: bool = False,
|
||||
) -> Module:
|
||||
if map_invalid_experts_to_sentinel and dtype not in (torch.int32, torch.int64):
|
||||
raise ValueError("Checked alignment requires int32 or int64 expert IDs")
|
||||
args = make_cpp_args(
|
||||
dtype,
|
||||
write_inactive_prefix_count,
|
||||
map_invalid_experts_to_sentinel,
|
||||
pad_expert_ids,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_align_block_size",
|
||||
*args,
|
||||
@@ -33,10 +45,23 @@ def moe_align_block_size(
|
||||
cumsum_buffer: torch.Tensor,
|
||||
pad_sorted_token_ids: bool = False,
|
||||
*,
|
||||
write_inactive_prefix_count: bool = False,
|
||||
map_invalid_experts_to_sentinel: bool = False,
|
||||
pad_expert_ids: bool = False,
|
||||
) -> None:
|
||||
"""Align routes; optionally define unused expert-map capacity as -1."""
|
||||
module = _jit_moe_align_module(topk_ids.dtype, pad_expert_ids)
|
||||
"""Align +1-offset buckets, optionally publishing a padded-token/block pair.
|
||||
|
||||
``write_inactive_prefix_count`` requires an aligned int32[2] count output.
|
||||
``pad_expert_ids`` fills unused expert-map capacity with -1.
|
||||
IDs must be in ``[-1, num_experts - 2]`` unless
|
||||
``map_invalid_experts_to_sentinel`` maps other signed IDs into bucket zero.
|
||||
"""
|
||||
module = _jit_moe_align_module(
|
||||
topk_ids.dtype,
|
||||
write_inactive_prefix_count,
|
||||
map_invalid_experts_to_sentinel,
|
||||
pad_expert_ids,
|
||||
)
|
||||
module.moe_align_block_size(
|
||||
topk_ids,
|
||||
num_experts,
|
||||
|
||||
@@ -18,8 +18,8 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_align_single_token_module() -> Module:
|
||||
args = make_cpp_args(is_arch_support_pdl())
|
||||
def _jit_align_single_token_module(write_inactive_prefix_count: bool = False) -> Module:
|
||||
args = make_cpp_args(is_arch_support_pdl(), write_inactive_prefix_count)
|
||||
return load_jit(
|
||||
"moe_align_single_token",
|
||||
*args,
|
||||
@@ -30,20 +30,26 @@ def _jit_align_single_token_module() -> Module:
|
||||
|
||||
|
||||
def moe_align_single_token(
|
||||
topk_ids: torch.Tensor, block_size: int
|
||||
topk_ids: torch.Tensor,
|
||||
block_size: int,
|
||||
*,
|
||||
return_inactive_prefix_count: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""moe_align_block_size for a single token (distinct expert ids).
|
||||
|
||||
Returns (sorted_token_ids, expert_ids, num_tokens_post_padded) with the
|
||||
same layout as moe_align_block_size: experts ascending, one block per
|
||||
expert, padding value = numel.
|
||||
expert, padding value = numel. With ``return_inactive_prefix_count``,
|
||||
the count is int32 ``[padded_tokens, inactive_blocks]``.
|
||||
"""
|
||||
topk = topk_ids.shape[1]
|
||||
device = topk_ids.device
|
||||
sorted_ids = torch.empty((topk * block_size,), dtype=torch.int32, device=device)
|
||||
expert_ids = torch.empty((topk,), dtype=torch.int32, device=device)
|
||||
num_post = torch.empty((1,), dtype=torch.int32, device=device)
|
||||
_jit_align_single_token_module().run(
|
||||
num_post = torch.empty(
|
||||
(2 if return_inactive_prefix_count else 1,), dtype=torch.int32, device=device
|
||||
)
|
||||
_jit_align_single_token_module(return_inactive_prefix_count).run(
|
||||
topk_ids, sorted_ids, expert_ids, num_post, block_size
|
||||
)
|
||||
return sorted_ids, expert_ids, num_post
|
||||
|
||||
@@ -24,13 +24,14 @@ def _moe_align_small_numel_kernel(
|
||||
topk_ids_ptr, # [numel] int, flattened (token, slot) expert ids, -1 = filtered
|
||||
sorted_token_ids_ptr, # [max_num_tokens_padded] int32
|
||||
expert_ids_ptr, # [max_num_m_blocks] int32
|
||||
num_tokens_post_pad_ptr, # [1] int32
|
||||
num_tokens_post_pad_ptr, # [1] or [2] int32
|
||||
num_experts, # E + 1 (the "+1 offset" convention's bucket count)
|
||||
block_size,
|
||||
numel,
|
||||
NP: tl.constexpr, # power-of-2 >= numel
|
||||
NB: tl.constexpr, # power-of-2 >= max blocks used
|
||||
USE_GDC: tl.constexpr = False,
|
||||
WRITE_INACTIVE_PREFIX_COUNT: tl.constexpr = False,
|
||||
):
|
||||
"""Single-CTA moe_align for tiny batches with MANY experts.
|
||||
|
||||
@@ -77,6 +78,11 @@ def _moe_align_small_numel_kernel(
|
||||
|
||||
total = tl.sum(tl.where(is_rep, padded_cnt, 0), axis=0)
|
||||
tl.store(num_tokens_post_pad_ptr, total.to(tl.int32))
|
||||
if WRITE_INACTIVE_PREFIX_COUNT:
|
||||
inactive_blocks = tl.sum(
|
||||
tl.where(is_rep & (bucket == 0), padded_cnt // block_size, 0), axis=0
|
||||
)
|
||||
tl.store(num_tokens_post_pad_ptr + 1, inactive_blocks.to(tl.int32))
|
||||
|
||||
# expert_ids per used block: representative r owns blocks
|
||||
# [excl[r], excl[r] + padded_cnt[r]); the written id is bucket - 1
|
||||
@@ -118,6 +124,8 @@ def moe_align_small_numel(
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_pad: torch.Tensor,
|
||||
*,
|
||||
write_inactive_prefix_count: bool = False,
|
||||
) -> None:
|
||||
"""Align and sort expert token ids into block-padded buffers, in one launch.
|
||||
|
||||
@@ -130,6 +138,12 @@ def moe_align_small_numel(
|
||||
correct above it, but its pairwise tensors spill and it stops being faster
|
||||
than the two-kernel path.
|
||||
"""
|
||||
if write_inactive_prefix_count:
|
||||
assert num_tokens_post_pad.shape == (2,)
|
||||
assert num_tokens_post_pad.dtype == torch.int32
|
||||
assert num_tokens_post_pad.device == topk_ids.device
|
||||
assert num_tokens_post_pad.is_contiguous()
|
||||
assert num_tokens_post_pad.data_ptr() % 8 == 0
|
||||
numel = topk_ids.numel()
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
|
||||
_moe_align_small_numel_kernel[(1,)](
|
||||
@@ -142,6 +156,7 @@ def moe_align_small_numel(
|
||||
numel,
|
||||
NP=triton.next_power_of_2(max(numel, 2)),
|
||||
NB=triton.next_power_of_2(max(expert_ids.numel(), 2)),
|
||||
WRITE_INACTIVE_PREFIX_COUNT=write_inactive_prefix_count,
|
||||
num_warps=4,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
@@ -17,9 +17,21 @@ _MAX_THREAD_N = 256
|
||||
|
||||
@cache_once
|
||||
def _jit_moe_wna16_marlin_module(
|
||||
dtype: torch.dtype, is_ep: bool, has_bias: bool
|
||||
dtype: torch.dtype,
|
||||
is_ep: bool,
|
||||
has_bias: bool,
|
||||
expert_ids_are_sorted: bool = False,
|
||||
zero_inactive_output: bool = False,
|
||||
has_inactive_prefix_count: bool = False,
|
||||
) -> Module:
|
||||
args = make_cpp_args(dtype, is_ep, has_bias)
|
||||
args = make_cpp_args(
|
||||
dtype,
|
||||
is_ep,
|
||||
has_bias,
|
||||
expert_ids_are_sorted,
|
||||
zero_inactive_output,
|
||||
has_inactive_prefix_count,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_wna16_marlin",
|
||||
*args,
|
||||
@@ -67,10 +79,22 @@ def moe_wna16_marlin_gemm(
|
||||
use_atomic_add: bool = False,
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
expert_ids_are_sorted: bool = False,
|
||||
zero_inactive_output: bool = False,
|
||||
has_inactive_prefix_count: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Project routes, preserving inactive rows in a caller-provided output.
|
||||
|
||||
Inactive rows are zero when this function allocates the output.
|
||||
Inactive rows are zero when this function allocates the output. With
|
||||
``zero_inactive_output=True``, the GEMM also zeros caller-owned rows present
|
||||
in inactive routing blocks; omitted routes remain untouched.
|
||||
``expert_ids_are_sorted`` certifies ascending block expert IDs within the
|
||||
padded token count, with any -1 blocks first. Unused tail entries are ignored.
|
||||
Leave it false after remapping that can produce scattered inactive blocks.
|
||||
The sorted specialization is only used without expert parallelism.
|
||||
``has_inactive_prefix_count`` additionally certifies that a supporting
|
||||
producer wrote an aligned int32 pair ``[padded_tokens, inactive_blocks]``
|
||||
for these tables and this block size. It is ignored outside sorted non-EP.
|
||||
"""
|
||||
device = a.device
|
||||
|
||||
@@ -140,7 +164,14 @@ def moe_wna16_marlin_gemm(
|
||||
b_bias_t = _or_empty(b_bias_or_none, device, a.dtype)
|
||||
global_scale_t = _or_empty(global_scale_or_none, device, a.dtype)
|
||||
|
||||
module = _jit_moe_wna16_marlin_module(a.dtype, is_ep, has_bias)
|
||||
module = _jit_moe_wna16_marlin_module(
|
||||
a.dtype,
|
||||
is_ep,
|
||||
has_bias,
|
||||
expert_ids_are_sorted and not is_ep,
|
||||
zero_inactive_output,
|
||||
has_inactive_prefix_count and expert_ids_are_sorted and not is_ep,
|
||||
)
|
||||
module.moe_wna16_marlin_gemm(
|
||||
a,
|
||||
c,
|
||||
|
||||
@@ -21,6 +21,7 @@ def _post_expert_norm_weighted_sum_kernel(
|
||||
HAS_WEIGHT: tl.constexpr,
|
||||
WEIGHT_FIRST: tl.constexpr,
|
||||
HAS_IDS: tl.constexpr,
|
||||
ROUND_PRODUCT: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
):
|
||||
tok = tl.program_id(0)
|
||||
@@ -55,7 +56,10 @@ def _post_expert_norm_weighted_sum_kernel(
|
||||
y = x
|
||||
|
||||
s = tl.load(s_ptr + tok * TOPK + k).to(tl.float32)
|
||||
acc += y * s
|
||||
product = y * s
|
||||
if ROUND_PRODUCT:
|
||||
product = product.to(x_ptr.dtype.element_ty).to(tl.float32)
|
||||
acc += product
|
||||
|
||||
tl.store(out_ptr + tok * H + offs, acc.to(out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
@@ -68,20 +72,20 @@ def fused_post_expert_norm_weighted_sum(
|
||||
weight_first: bool = True,
|
||||
apply_norm: bool = True,
|
||||
slot_expert_ids: Optional[torch.Tensor] = None,
|
||||
round_product: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize and combine experts without reading inactive slots.
|
||||
|
||||
round_product preserves input-dtype multiplication before FP32 accumulation
|
||||
when combining an already normalized low-precision tensor.
|
||||
"""
|
||||
x = expert_output.contiguous()
|
||||
tokens, topk, hidden = x.shape
|
||||
out = torch.empty((tokens, hidden), dtype=x.dtype, device=x.device)
|
||||
if tokens == 0:
|
||||
return out
|
||||
if tokens < _MIN_TOKENS_FOR_FUSED:
|
||||
if tokens < _MIN_TOKENS_FOR_FUSED and slot_expert_ids is None and not round_product:
|
||||
y = x
|
||||
if slot_expert_ids is not None:
|
||||
y = torch.where(
|
||||
(slot_expert_ids.reshape(tokens, topk) >= 0).unsqueeze(-1),
|
||||
y,
|
||||
torch.zeros((), dtype=y.dtype, device=y.device),
|
||||
)
|
||||
if apply_norm:
|
||||
f = y.float()
|
||||
if norm_weight is not None and weight_first:
|
||||
@@ -101,10 +105,10 @@ def fused_post_expert_norm_weighted_sum(
|
||||
block_h = triton.next_power_of_2(hidden)
|
||||
_post_expert_norm_weighted_sum_kernel[(tokens,)](
|
||||
x,
|
||||
norm_weight if norm_weight is not None else x,
|
||||
norm_weight.contiguous() if norm_weight is not None else x,
|
||||
topk_scores.contiguous(),
|
||||
out,
|
||||
slot_expert_ids if slot_expert_ids is not None else x,
|
||||
slot_expert_ids.contiguous() if slot_expert_ids is not None else x,
|
||||
TOPK=topk,
|
||||
H=hidden,
|
||||
eps=eps,
|
||||
@@ -112,6 +116,7 @@ def fused_post_expert_norm_weighted_sum(
|
||||
HAS_WEIGHT=norm_weight is not None,
|
||||
WEIGHT_FIRST=weight_first,
|
||||
HAS_IDS=slot_expert_ids is not None,
|
||||
ROUND_PRODUCT=round_product,
|
||||
BLOCK_H=block_h,
|
||||
num_warps=8 if block_h >= 4096 else 4,
|
||||
num_stages=2,
|
||||
|
||||
@@ -275,11 +275,14 @@ def fused_marlin_moe(
|
||||
from sglang.kernels.ops.moe.moe_align_single_token import moe_align_single_token
|
||||
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_single_token(
|
||||
topk_ids, block_size_m
|
||||
topk_ids, block_size_m, return_inactive_prefix_count=True
|
||||
)
|
||||
else:
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, block_size_m, global_num_experts
|
||||
topk_ids,
|
||||
block_size_m,
|
||||
global_num_experts,
|
||||
return_inactive_prefix_count=expert_map is None,
|
||||
)
|
||||
|
||||
if workspace is None:
|
||||
@@ -339,6 +342,8 @@ def fused_marlin_moe(
|
||||
top_k=topk,
|
||||
mul_topk_weights=False,
|
||||
is_ep=expert_map is not None,
|
||||
expert_ids_are_sorted=expert_map is None,
|
||||
has_inactive_prefix_count=expert_map is None,
|
||||
b_q_type=scalar_type1,
|
||||
size_m=M,
|
||||
size_n=gemm1_n,
|
||||
@@ -380,9 +385,7 @@ def fused_marlin_moe(
|
||||
else:
|
||||
raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}")
|
||||
|
||||
# Gate-up is dead after activation, so its storage can hold the down output.
|
||||
# Skipped down routes must start at zero before the top-k reduction.
|
||||
intermediate_cache3.zero_()
|
||||
# Gate-up is dead; the down GEMM defines inactive rows in the shared output.
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
intermediate_cache3,
|
||||
@@ -401,7 +404,10 @@ def fused_marlin_moe(
|
||||
moe_block_size=block_size_m,
|
||||
top_k=1,
|
||||
mul_topk_weights=True,
|
||||
zero_inactive_output=True,
|
||||
is_ep=expert_map is not None,
|
||||
expert_ids_are_sorted=expert_map is None,
|
||||
has_inactive_prefix_count=expert_map is None,
|
||||
b_q_type=scalar_type2,
|
||||
size_m=M * topk,
|
||||
size_n=K,
|
||||
|
||||
@@ -35,6 +35,7 @@ def moe_align_block_size(
|
||||
num_experts: int,
|
||||
ignore_invalid_expert: bool = False,
|
||||
*,
|
||||
return_inactive_prefix_count: bool = False,
|
||||
pad_expert_ids: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
@@ -52,7 +53,9 @@ def moe_align_block_size(
|
||||
to their allocated expert.
|
||||
- expert_ids: A tensor indicating the assigned expert index for each block.
|
||||
- num_tokens_post_padded: The total number of tokens after padding,
|
||||
ensuring divisibility by block_size.
|
||||
ensuring divisibility by block_size. Opting into ``return_inactive_prefix_count``
|
||||
returns int32 ``[padded_tokens, inactive_blocks]`` from a supporting local
|
||||
producer; the second word counts negative physical blocks, not tokens.
|
||||
|
||||
This function pads the number of tokens that each expert needs to process
|
||||
so that it is divisible by block_size.
|
||||
@@ -79,6 +82,11 @@ def moe_align_block_size(
|
||||
"Padding expert IDs requires CUDA and the invalid-expert bucket"
|
||||
)
|
||||
|
||||
if return_inactive_prefix_count:
|
||||
if not _is_cuda or ignore_invalid_expert or num_experts > 8191:
|
||||
raise ValueError(
|
||||
"Inactive-prefix counts require CUDA alignment with at most 8191 experts and the +1 bucket convention"
|
||||
)
|
||||
# ===== TO BE REFACTORED ====
|
||||
if _SGLANG_EXPERIMENTAL_LORA_OPTI and not pad_expert_ids:
|
||||
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
|
||||
@@ -88,7 +96,20 @@ def moe_align_block_size(
|
||||
_align_block_size_jit,
|
||||
)
|
||||
|
||||
return _align_block_size_jit(topk_ids, block_size, num_experts)
|
||||
if return_inactive_prefix_count:
|
||||
return _align_block_size_jit(
|
||||
topk_ids,
|
||||
block_size,
|
||||
num_experts,
|
||||
return_inactive_prefix_count=return_inactive_prefix_count,
|
||||
map_invalid_experts_to_sentinel=False,
|
||||
)
|
||||
return _align_block_size_jit(
|
||||
topk_ids,
|
||||
block_size,
|
||||
num_experts,
|
||||
map_invalid_experts_to_sentinel=False,
|
||||
)
|
||||
# ===== END TO BE REFACTORED ====
|
||||
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
@@ -105,7 +126,11 @@ def moe_align_block_size(
|
||||
expert_ids = torch.empty(
|
||||
(max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device)
|
||||
num_tokens_post_pad = torch.empty(
|
||||
(2 if return_inactive_prefix_count else 1,),
|
||||
dtype=torch.int32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
|
||||
# In EP, expert_ids for filtered experts are -1. We have num_experts + 1 ids in total.
|
||||
cumsum_buffer = torch.empty(
|
||||
@@ -132,15 +157,18 @@ def moe_align_block_size(
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
write_inactive_prefix_count=return_inactive_prefix_count,
|
||||
)
|
||||
return sorted_ids, expert_ids, num_tokens_post_pad
|
||||
|
||||
# ===== TO BE REFACTORED ====
|
||||
use_jit_align = pad_expert_ids
|
||||
use_jit_align = return_inactive_prefix_count or pad_expert_ids
|
||||
if _SGLANG_EXPERIMENTAL_LORA_OPTI:
|
||||
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
|
||||
|
||||
use_jit_align |= lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get()
|
||||
use_jit_align = (
|
||||
use_jit_align or lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get()
|
||||
)
|
||||
if use_jit_align:
|
||||
from sglang.kernels.ops.moe.moe_align import (
|
||||
moe_align_block_size as jit_moe_align_block_size,
|
||||
@@ -155,6 +183,7 @@ def moe_align_block_size(
|
||||
num_tokens_post_pad,
|
||||
cumsum_buffer,
|
||||
True,
|
||||
write_inactive_prefix_count=return_inactive_prefix_count,
|
||||
pad_expert_ids=pad_expert_ids,
|
||||
)
|
||||
# ===== END TO BE REFACTORED ====
|
||||
|
||||
@@ -382,7 +382,8 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project local routes; finalization skips undefined inactive slots."""
|
||||
from sgl_kernel.scalar_type import scalar_types
|
||||
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
@@ -416,7 +417,9 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
)
|
||||
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = (
|
||||
moe_align_single_token(local_topk_ids, block_size_m)
|
||||
moe_align_single_token(
|
||||
local_topk_ids, block_size_m, return_inactive_prefix_count=True
|
||||
)
|
||||
)
|
||||
else:
|
||||
align_num_experts = (
|
||||
@@ -425,22 +428,27 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
else self.num_local_experts
|
||||
)
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
local_topk_ids, block_size_m, align_num_experts
|
||||
local_topk_ids,
|
||||
block_size_m,
|
||||
align_num_experts,
|
||||
return_inactive_prefix_count=expert_map is None,
|
||||
)
|
||||
|
||||
# Both aligners emit ascending block IDs with inactive blocks first;
|
||||
# no expert remapping occurs after this ordering is established.
|
||||
workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
|
||||
cache2 = torch.empty(
|
||||
(num_tokens * topk, size_n),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache13 = torch.zeros(
|
||||
(num_tokens * topk * (gemm1_n + size_k),),
|
||||
cache13 = torch.empty(
|
||||
(num_tokens * topk * max(gemm1_n, size_k),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
|
||||
cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k)
|
||||
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
|
||||
|
||||
cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
@@ -469,6 +477,8 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
expert_ids_are_sorted=True,
|
||||
has_inactive_prefix_count=expert_map is None,
|
||||
)
|
||||
silu_and_mul(
|
||||
cache1.view(-1, gemm1_n),
|
||||
@@ -503,6 +513,8 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
expert_ids_are_sorted=True,
|
||||
has_inactive_prefix_count=expert_map is None,
|
||||
).view(num_tokens, topk, size_k), local_topk_ids
|
||||
|
||||
def _forward_marlin(
|
||||
@@ -529,7 +541,6 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
|
||||
hidden_states, topk_scores, topk_ids
|
||||
)
|
||||
topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0)
|
||||
if self._post_expert_norm is not post_expert_norm:
|
||||
self.set_post_expert_norm(post_expert_norm)
|
||||
if self._post_expert_norm_weight is not None:
|
||||
@@ -546,6 +557,7 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
topk_scores=topk_scores,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=hidden_states.dtype,
|
||||
slot_expert_ids=local_topk_ids,
|
||||
)
|
||||
|
||||
def prepare_input(
|
||||
@@ -756,10 +768,20 @@ def finalize_identity_with_post_expert_norm(
|
||||
topk_scores: torch.Tensor,
|
||||
post_expert_norm: nn.Module,
|
||||
out_dtype: torch.dtype,
|
||||
slot_expert_ids: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply row-local expert normalization before combining live routes."""
|
||||
num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1]
|
||||
normed = post_expert_norm(expert_outputs)
|
||||
weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype)
|
||||
if slot_expert_ids is not None:
|
||||
return fused_post_expert_norm_weighted_sum(
|
||||
normed,
|
||||
weights,
|
||||
apply_norm=False,
|
||||
slot_expert_ids=slot_expert_ids,
|
||||
round_product=True,
|
||||
).to(out_dtype)
|
||||
return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user