#pragma once #include "vec.h" template inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) { using Vec = at::vec::Vectorized; const Vec data_vec(val); at::vec::map([data_vec](Vec out) { return out = data_vec; }, out, out, size); } template inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { using Vec = at::vec::Vectorized; constexpr int kVecSize = Vec::size(); int64_t d; #pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { Vec data = Vec::loadu(input + d); data.store(out + d); } for (; d < size; ++d) { out[d] = input[d]; } } template inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); int64_t d; #pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { auto [x0, x1] = load_float_vec2(input + d); bVec out_vec = convert_from_float_ext(x0, x1); out_vec.store(out + d); } for (; d < size; ++d) { out[d] = static_cast(input[d]); } } template <> inline void copy_stub(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) { // size might be 64x + 32 std::memcpy(out, input, size * sizeof(uint8_t)); } template inline void copy_mul_stub(scalar_t* __restrict__ out, const input_t* __restrict__ input, float weight, int64_t size) { static_assert( std::is_same_v || std::is_same_v, "copy_mul_stub only supports input_t == float or input_t == scalar_t"); using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); const fVec weight_vec = fVec(weight); int64_t d; #pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { auto [x0, x1] = load_float_vec2(input + d); x0 = x0 * weight_vec; x1 = x1 * weight_vec; bVec out_vec = convert_from_float_ext(x0, x1); out_vec.store(out + d); } for (; d < size; ++d) { out[d] = static_cast(input[d] * weight); } } // acc from [topk, K] to [K] template inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); if (topk == 1) { // do copy for topk = 1 copy_stub(out, input, K); } else { // do sum for topk != 1 int64_t d; #pragma GCC unroll 4 for (d = 0; d <= K - kVecSize; d += kVecSize) { fVec sum_fvec0 = fVec(0.f); fVec sum_fvec1 = fVec(0.f); for (int t = 0; t < topk; ++t) { bVec x_bvec = bVec::loadu(input + t * K + d); fVec x_fvec0, x_fvec1; std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); sum_fvec0 += x_fvec0; sum_fvec1 += x_fvec1; } bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); out_bvec.store(out + d); } for (; d < K; ++d) { float sum_val = 0.f; for (int t = 0; t < topk; ++t) { sum_val += static_cast(input[t * K + d]); } out[d] = static_cast(sum_val); } } } // out = input + input2 * scale template inline void add_mul_stub( scalar_t* __restrict__ out, const input_t* __restrict__ input, const scalar_t* __restrict__ input2, float scale, int64_t size) { static_assert( std::is_same_v || std::is_same_v, "add_mul_stub only supports input_t == float or input_t == scalar_t"); // out = input (without scale factor) if (input2 == nullptr) { copy_stub(out, input, size); return; } using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); const fVec s_vec = fVec(scale); int64_t d; #pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { auto [x0, x1] = load_float_vec2(input + d); bVec y_bvec = bVec::loadu(input2 + d); fVec y0, y1; std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); x0 = x0 + y0 * s_vec; x1 = x1 + y1 * s_vec; bVec out_vec = convert_from_float_ext(x0, x1); out_vec.store(out + d); } for (; d < size; ++d) { out[d] = static_cast(input[d] + float(input2[d]) * scale); } } template inline void silu_and_mul_stub( scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; const fVec one = fVec(1.f); // no remainder #pragma GCC unroll 4 for (int64_t d = 0; d < size; d += bVec::size()) { bVec x = bVec::loadu(input + d); fVec x0, x1; std::tie(x0, x1) = at::vec::convert_to_float(x); bVec y = bVec::loadu(input2 + d); fVec y0, y1; std::tie(y0, y1) = at::vec::convert_to_float(y); x0 = x0 / (one + x0.neg().exp_u20()); x1 = x1 / (one + x1.neg().exp_u20()); x0 = x0 * y0; x1 = x1 * y1; bVec out_vec = convert_from_float_ext(x0, x1); out_vec.store(out + d); } }