[Fix] lora: Preserve MoE projection identity and routing lifetimes (#45681)
This commit is contained in:
+108
-43
@@ -48,11 +48,21 @@ using Vec = int4;
|
||||
|
||||
namespace moe_lora_merged {
|
||||
|
||||
template <bool HasAdapterEnabled>
|
||||
__device__ __forceinline__ bool
|
||||
adapter_is_enabled(int lora_id, const int32_t* adapter_enabled, int num_slots, int64_t enabled_stride) {
|
||||
if (lora_id < 0 || lora_id >= num_slots) return false;
|
||||
if constexpr (HasAdapterEnabled) {
|
||||
return adapter_enabled[lora_id * enabled_stride] != 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Inline mirror of _fused_virtual_topk_ids_kernel (virtual_experts.py). Returns
|
||||
// the merged virtual expert id for flat slot `i` (range [-1, virtual_num_experts);
|
||||
// -1 is the dropped/masked sentinel). The caller adds +1 to get the histogram
|
||||
// bucket (sentinel -> bucket 0), matching the native +1 offset convention.
|
||||
template <typename scalar_t>
|
||||
template <typename scalar_t, bool HasAdapterEnabled>
|
||||
__device__ __forceinline__ int compute_virtual_id(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
const int32_t* __restrict__ token_lora_mapping,
|
||||
@@ -63,27 +73,32 @@ __device__ __forceinline__ int compute_virtual_id(
|
||||
int local_num_experts,
|
||||
bool ep_local,
|
||||
bool shared_outer,
|
||||
bool compact) {
|
||||
bool compact,
|
||||
const int32_t* __restrict__ adapter_enabled,
|
||||
int num_slots,
|
||||
int64_t enabled_stride) {
|
||||
int m = static_cast<int>(i) / top_k;
|
||||
int lora_id = token_lora_mapping[m];
|
||||
bool mask_val = lora_id >= 0;
|
||||
int safe_lora = lora_id > 0 ? lora_id : 0;
|
||||
bool mask_val = adapter_is_enabled<HasAdapterEnabled>(lora_id, adapter_enabled, num_slots, enabled_stride);
|
||||
|
||||
int base = shared_outer ? 0 : static_cast<int>(topk_ids[i]);
|
||||
scalar_t original_id = topk_ids[i];
|
||||
if (!mask_val || original_id < 0) return -1;
|
||||
if (!shared_outer && original_id >= num_experts_for_weight) return -1;
|
||||
int base = shared_outer ? 0 : static_cast<int>(original_id);
|
||||
if (ep_local) {
|
||||
bool owned = base >= local_expert_offset && base < local_expert_offset + local_num_experts;
|
||||
base = owned ? base : -1;
|
||||
}
|
||||
if (!mask_val || base < 0) return -1;
|
||||
if (base < 0) return -1;
|
||||
// compact: dense LOCAL expert id in [0, local_num_experts) so the histogram
|
||||
// spans only local_num_experts buckets instead of the full global virtual
|
||||
// space (337/385 empty under EP). Assumes max_loras==1 (safe_lora shift is 0;
|
||||
// space (337/385 empty under EP). Assumes max_loras==1 (adapter shift is 0;
|
||||
// the wrapper guards). expert_ids is converted back to global at write time.
|
||||
if (compact) return base - local_expert_offset;
|
||||
return base + safe_lora * num_experts_for_weight;
|
||||
return base + lora_id * num_experts_for_weight;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
template <typename scalar_t, bool HasAdapterEnabled>
|
||||
__global__ void count_and_sort_expert_tokens_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
const int32_t* __restrict__ token_lora_mapping,
|
||||
@@ -97,12 +112,15 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
bool ep_local,
|
||||
bool shared_outer,
|
||||
bool do_skip,
|
||||
bool compact) {
|
||||
bool compact,
|
||||
const int32_t* __restrict__ adapter_enabled,
|
||||
int num_slots,
|
||||
int64_t enabled_stride) {
|
||||
const size_t tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const size_t stride = blockDim.x * gridDim.x;
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int vid = compute_virtual_id<scalar_t>(
|
||||
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
i,
|
||||
@@ -112,7 +130,10 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
local_num_experts,
|
||||
ep_local,
|
||||
shared_outer,
|
||||
compact);
|
||||
compact,
|
||||
adapter_enabled,
|
||||
num_slots,
|
||||
enabled_stride);
|
||||
// EP skip: dropped/masked slots (vid < 0) produce no delta on this rank, so
|
||||
// they never need a slot in sorted_token_ids -> skip the global atomicAdd
|
||||
// (kills the sentinel-bucket-0 contention). When do_skip is off they fall
|
||||
@@ -124,7 +145,7 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
|
||||
__global__ void moe_align_block_size_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
const int32_t* __restrict__ token_lora_mapping,
|
||||
@@ -146,7 +167,10 @@ __global__ void moe_align_block_size_kernel(
|
||||
bool ep_local,
|
||||
bool shared_outer,
|
||||
bool do_skip,
|
||||
bool compact) {
|
||||
bool compact,
|
||||
const int32_t* __restrict__ adapter_enabled,
|
||||
int num_slots,
|
||||
int64_t enabled_stride) {
|
||||
// Use a separate thread block to populate sorted_token_ids
|
||||
if (blockIdx.x == 1) {
|
||||
if (pad_sorted_token_ids) {
|
||||
@@ -181,7 +205,7 @@ __global__ void moe_align_block_size_kernel(
|
||||
__syncthreads();
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int vid = compute_virtual_id<scalar_t>(
|
||||
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
i,
|
||||
@@ -191,16 +215,20 @@ __global__ void moe_align_block_size_kernel(
|
||||
local_num_experts,
|
||||
ep_local,
|
||||
shared_outer,
|
||||
compact);
|
||||
compact,
|
||||
adapter_enabled,
|
||||
num_slots,
|
||||
enabled_stride);
|
||||
// EP skip: dropped/masked slots don't increment any bucket (sentinel bucket
|
||||
// 0 stays empty), so they never get a block and never reach count_and_sort.
|
||||
if (!do_skip || vid >= 0) {
|
||||
atomicAdd(&shared_counts[vid + 1], 1);
|
||||
}
|
||||
// token_lora_mask[m] = token_lora_mapping[m] >= 0, written once per row.
|
||||
// Publish the same bounded adapter predicate used by both routing passes.
|
||||
if (static_cast<int>(i) % top_k == 0) {
|
||||
int m = static_cast<int>(i) / top_k;
|
||||
token_lora_mask[m] = token_lora_mapping[m] >= 0;
|
||||
token_lora_mask[m] =
|
||||
adapter_is_enabled<HasAdapterEnabled>(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,6 +263,9 @@ __global__ void moe_align_block_size_kernel(
|
||||
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
if constexpr (WriteInactivePrefixCount) {
|
||||
total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -277,9 +308,9 @@ __global__ void moe_align_block_size_kernel(
|
||||
right = mid;
|
||||
}
|
||||
}
|
||||
// compact buckets hold LOCAL expert ids; restore the global id (+offset) so
|
||||
// the downstream GEMM still indexes the global contiguous LoRA weight.
|
||||
expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0);
|
||||
// Restore global IDs only for real compact buckets. The negative bucket
|
||||
// stays -1 so overwrite consumers can zero it before touching weights.
|
||||
expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,9 +318,9 @@ __global__ void moe_align_block_size_kernel(
|
||||
// in ONE threadblock (one launch), eliminating the separate count_and_sort kernel
|
||||
// AND its redundant re-computation of the virtual id (cached in shared `svids`).
|
||||
// Only valid for small numel (the scatter is single-block); the wrapper routes
|
||||
// large numel (prefill) to the 2-kernel path. do_skip is implied (this is the
|
||||
// decode hot path); dropped slots are simply never scattered.
|
||||
template <typename scalar_t>
|
||||
// large numel (prefill) to the 2-kernel path. do_skip omits dropped slots;
|
||||
// overwrite consumers can retain them for the existing negative-expert store.
|
||||
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
|
||||
__global__ void fused_align_scatter_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
const int32_t* __restrict__ token_lora_mapping,
|
||||
@@ -310,7 +341,10 @@ __global__ void fused_align_scatter_kernel(
|
||||
bool ep_local,
|
||||
bool shared_outer,
|
||||
bool do_skip,
|
||||
bool compact) {
|
||||
bool compact,
|
||||
const int32_t* __restrict__ adapter_enabled,
|
||||
int num_slots,
|
||||
int64_t enabled_stride) {
|
||||
extern __shared__ int32_t smem[];
|
||||
int32_t* shared_counts = smem; // [num_experts]
|
||||
int32_t* prefix = shared_counts + num_experts; // [num_experts + 1]
|
||||
@@ -345,7 +379,7 @@ __global__ void fused_align_scatter_kernel(
|
||||
|
||||
// Phase 2: histogram + cache the virtual id per slot + token_lora_mask.
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int vid = compute_virtual_id<scalar_t>(
|
||||
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
i,
|
||||
@@ -355,14 +389,18 @@ __global__ void fused_align_scatter_kernel(
|
||||
local_num_experts,
|
||||
ep_local,
|
||||
shared_outer,
|
||||
compact);
|
||||
compact,
|
||||
adapter_enabled,
|
||||
num_slots,
|
||||
enabled_stride);
|
||||
svids[i] = vid;
|
||||
if (!do_skip || vid >= 0) {
|
||||
atomicAdd(&shared_counts[vid + 1], 1);
|
||||
}
|
||||
if (static_cast<int>(i) % top_k == 0) {
|
||||
int m = static_cast<int>(i) / top_k;
|
||||
token_lora_mask[m] = token_lora_mapping[m] >= 0;
|
||||
token_lora_mask[m] =
|
||||
adapter_is_enabled<HasAdapterEnabled>(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
@@ -387,6 +425,9 @@ __global__ void fused_align_scatter_kernel(
|
||||
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
if constexpr (WriteInactivePrefixCount) {
|
||||
total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0;
|
||||
@@ -420,7 +461,7 @@ __global__ void fused_align_scatter_kernel(
|
||||
right = mid;
|
||||
}
|
||||
}
|
||||
expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0);
|
||||
expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0);
|
||||
}
|
||||
if (tid < num_experts) cursor[tid] = prefix[tid];
|
||||
__syncthreads();
|
||||
@@ -437,7 +478,7 @@ __global__ void fused_align_scatter_kernel(
|
||||
|
||||
} // namespace moe_lora_merged
|
||||
|
||||
template <typename scalar_t>
|
||||
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
|
||||
struct MoeLoraMergedAlignKernel {
|
||||
static void
|
||||
run(tvm::ffi::TensorView topk_ids,
|
||||
@@ -458,7 +499,10 @@ struct MoeLoraMergedAlignKernel {
|
||||
bool shared_outer,
|
||||
bool do_skip,
|
||||
bool compact,
|
||||
bool fuse_scatter) {
|
||||
bool fuse_scatter,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> adapter_enabled,
|
||||
int64_t num_slots,
|
||||
int64_t enabled_stride) {
|
||||
using namespace host;
|
||||
|
||||
auto device = topk_ids.device();
|
||||
@@ -475,12 +519,6 @@ struct MoeLoraMergedAlignKernel {
|
||||
// shared memory); the v2 (>1024) regime keeps the old path via the wrapper.
|
||||
RuntimeCheck(
|
||||
num_experts <= 1024, "moe_lora_merged_align: num_experts (bucket count) must be <= 1024, got ", num_experts);
|
||||
// compact buckets hold LOCAL ids and restore global expert ids as
|
||||
// (left-2+offset). For the sentinel bucket 0 that yields (offset-1), NOT the
|
||||
// -1 the GEMM expects to skip -- only safe when do_skip empties bucket 0.
|
||||
RuntimeCheck(
|
||||
!compact || do_skip, "moe_lora_merged_align: compact requires do_skip (sentinel bucket must be empty)");
|
||||
|
||||
const scalar_t* topk_ids_ptr = static_cast<const scalar_t*>(topk_ids.data_ptr());
|
||||
const int32_t* tlm_ptr = static_cast<const int32_t*>(token_lora_mapping.data_ptr());
|
||||
bool* token_lora_mask_ptr = static_cast<bool*>(token_lora_mask.data_ptr());
|
||||
@@ -489,6 +527,21 @@ struct MoeLoraMergedAlignKernel {
|
||||
int32_t* num_tokens_post_pad_ptr = static_cast<int32_t*>(num_tokens_post_pad.data_ptr());
|
||||
int32_t* cumsum_buffer_ptr = static_cast<int32_t*>(cumsum_buffer.data_ptr());
|
||||
size_t numel = topk_ids.numel();
|
||||
const int32_t* enabled_ptr = nullptr;
|
||||
if constexpr (HasAdapterEnabled) {
|
||||
CHECK_HOST(adapter_enabled.has_value());
|
||||
TensorMatcher({num_slots})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.with_strides({enabled_stride})
|
||||
.verify(adapter_enabled.value());
|
||||
enabled_ptr = static_cast<const int32_t*>(adapter_enabled.value().data_ptr());
|
||||
}
|
||||
TensorMatcher({WriteInactivePrefixCount ? 2 : 1})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(WriteInactivePrefixCount ? 8 : 4)
|
||||
.verify(num_tokens_post_pad);
|
||||
|
||||
const size_t scan_size = host::round_up_pow2(static_cast<uint32_t>(num_experts));
|
||||
|
||||
@@ -497,7 +550,7 @@ struct MoeLoraMergedAlignKernel {
|
||||
// shared for the scatter cursor [num_experts] and cached virtual ids [numel].
|
||||
const size_t shmem =
|
||||
(num_experts + (num_experts + 1) + scan_size + WARP_SIZE + num_experts + numel) * sizeof(int32_t);
|
||||
auto fused = moe_lora_merged::fused_align_scatter_kernel<scalar_t>;
|
||||
auto fused = moe_lora_merged::fused_align_scatter_kernel<scalar_t, HasAdapterEnabled, WriteInactivePrefixCount>;
|
||||
LaunchKernel(dim3(1), dim3(threads), stream, shmem)(
|
||||
fused,
|
||||
topk_ids_ptr,
|
||||
@@ -519,13 +572,17 @@ struct MoeLoraMergedAlignKernel {
|
||||
ep_local,
|
||||
shared_outer,
|
||||
do_skip,
|
||||
compact);
|
||||
compact,
|
||||
enabled_ptr,
|
||||
static_cast<int>(num_slots),
|
||||
enabled_stride);
|
||||
return;
|
||||
}
|
||||
|
||||
const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t);
|
||||
|
||||
auto align_kernel = moe_lora_merged::moe_align_block_size_kernel<scalar_t>;
|
||||
auto align_kernel =
|
||||
moe_lora_merged::moe_align_block_size_kernel<scalar_t, HasAdapterEnabled, WriteInactivePrefixCount>;
|
||||
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
|
||||
align_kernel,
|
||||
topk_ids_ptr,
|
||||
@@ -548,14 +605,19 @@ struct MoeLoraMergedAlignKernel {
|
||||
ep_local,
|
||||
shared_outer,
|
||||
do_skip,
|
||||
compact);
|
||||
compact,
|
||||
enabled_ptr,
|
||||
static_cast<int>(num_slots),
|
||||
enabled_stride);
|
||||
|
||||
if (numel == 0) return;
|
||||
|
||||
const int block_threads = std::min(256, threads);
|
||||
const int num_blocks = (numel + block_threads - 1) / block_threads;
|
||||
const int max_blocks = 65535;
|
||||
const int actual_blocks = std::min(num_blocks, max_blocks);
|
||||
|
||||
auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel<scalar_t, HasAdapterEnabled>;
|
||||
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
|
||||
sort_kernel,
|
||||
topk_ids_ptr,
|
||||
@@ -570,7 +632,10 @@ struct MoeLoraMergedAlignKernel {
|
||||
ep_local,
|
||||
shared_outer,
|
||||
do_skip,
|
||||
compact);
|
||||
compact,
|
||||
enabled_ptr,
|
||||
static_cast<int>(num_slots),
|
||||
enabled_stride);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -50,18 +50,17 @@ def route_bucket_ids(
|
||||
# Invalid slots/groups map to -1. With one group per slot, any nonnegative
|
||||
# group folds into that slot; otherwise bound it to the bucket table.
|
||||
token_ids = pair_ids // WIDTH
|
||||
slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1).to(tl.int32)
|
||||
slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1)
|
||||
live = (slots >= 0) & (slots < MAX_LORAS)
|
||||
slots = slots.to(tl.int32)
|
||||
if HAS_GROUPS:
|
||||
groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1).to(
|
||||
tl.int32
|
||||
)
|
||||
groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1)
|
||||
live = live & (groups >= 0)
|
||||
if GROUPS_PER_SLOT == 1:
|
||||
buckets = slots
|
||||
else:
|
||||
live = live & (groups < GROUPS_PER_SLOT)
|
||||
buckets = slots * GROUPS_PER_SLOT + groups
|
||||
buckets = slots * GROUPS_PER_SLOT + groups.to(tl.int32)
|
||||
else:
|
||||
buckets = slots
|
||||
return tl.where(live, buckets, -1)
|
||||
|
||||
@@ -11,8 +11,16 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module(dtype: torch.dtype) -> Module:
|
||||
args = make_cpp_args(dtype)
|
||||
def _jit_module(
|
||||
dtype: torch.dtype,
|
||||
has_adapter_enabled: bool = False,
|
||||
write_inactive_prefix_count: bool = False,
|
||||
) -> Module:
|
||||
args = make_cpp_args(
|
||||
dtype,
|
||||
has_adapter_enabled,
|
||||
write_inactive_prefix_count,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_lora_merged_align",
|
||||
*args,
|
||||
@@ -35,6 +43,8 @@ def moe_lora_merged_align(
|
||||
do_skip: bool = True,
|
||||
compact: bool = False,
|
||||
fuse_scatter: Optional[bool] = None,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
return_inactive_prefix_count: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
"""Fused replacement for (_fused_virtual_topk_ids + _align_block_size) on the
|
||||
merged-virtual-expert LoRA path.
|
||||
@@ -44,13 +54,17 @@ def moe_lora_merged_align(
|
||||
|
||||
Returns (sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
token_lora_mask, virtual_num_experts).
|
||||
With ``return_inactive_prefix_count``, the count is an aligned int32 pair
|
||||
[total padded tokens, leading negative-expert blocks]; do_skip publishes zero.
|
||||
"""
|
||||
device = topk_ids.device
|
||||
flat_topk_ids = topk_ids.reshape(-1)
|
||||
if flat_topk_ids.dtype == torch.int64:
|
||||
flat_topk_ids = flat_topk_ids.to(torch.int32)
|
||||
M, top_k = topk_ids.shape
|
||||
numel = M * top_k
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (max_loras,)
|
||||
assert adapter_enabled.dtype == torch.int32
|
||||
assert adapter_enabled.device == device
|
||||
|
||||
num_experts_for_weight = 1 if shared_outer else num_experts
|
||||
virtual_num_experts = num_experts_for_weight * max_loras
|
||||
@@ -97,13 +111,19 @@ def moe_lora_merged_align(
|
||||
sorted_alloc = (max_num_tokens_padded + 3) & ~3
|
||||
sorted_token_ids = torch.empty((sorted_alloc,), dtype=torch.int32, device=device)
|
||||
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
|
||||
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
|
||||
num_tokens_post_pad = torch.empty(
|
||||
(2 if return_inactive_prefix_count else 1,), dtype=torch.int32, device=device
|
||||
)
|
||||
# No memset: the align kernel writes cumsum[0..num_buckets] before
|
||||
# count_and_sort reads it (same as moe_align_block_size.py's empty cumsum).
|
||||
cumsum_buffer = torch.empty((bucket_experts + 2,), dtype=torch.int32, device=device)
|
||||
token_lora_mask = torch.empty((M,), dtype=torch.bool, device=device)
|
||||
|
||||
module = _jit_module(flat_topk_ids.dtype)
|
||||
module = _jit_module(
|
||||
flat_topk_ids.dtype,
|
||||
adapter_enabled is not None,
|
||||
return_inactive_prefix_count,
|
||||
)
|
||||
module.moe_lora_merged_align(
|
||||
flat_topk_ids,
|
||||
token_lora_mapping,
|
||||
@@ -124,6 +144,9 @@ def moe_lora_merged_align(
|
||||
do_skip,
|
||||
compact_eff,
|
||||
fuse_eff,
|
||||
adapter_enabled,
|
||||
max_loras,
|
||||
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
|
||||
)
|
||||
|
||||
return (
|
||||
|
||||
@@ -22,26 +22,25 @@ def _fused_virtual_topk_ids_kernel(
|
||||
token_lora_mapping_ptr,
|
||||
virtual_topk_ids_ptr,
|
||||
token_lora_mask_ptr,
|
||||
adapter_enabled_ptr,
|
||||
stride_enabled,
|
||||
num_experts_for_weight: tl.constexpr,
|
||||
SHARED_OUTER: tl.constexpr,
|
||||
M,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
local_expert_offset: tl.constexpr,
|
||||
local_num_experts: tl.constexpr,
|
||||
EP_LOCAL: tl.constexpr,
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
HAS_ADAPTER_ENABLED: tl.constexpr,
|
||||
ENABLE_PDL: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel.
|
||||
Map active slots and valid base experts to virtual weight indices.
|
||||
|
||||
For each (m, k):
|
||||
lora_id = token_lora_mapping[m]
|
||||
mask[m] = (lora_id >= 0)
|
||||
safe_lora = max(lora_id, 0)
|
||||
if shared_outer: (handled by num_experts_for_weight == 0 sentinel)
|
||||
virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora)
|
||||
else:
|
||||
virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight
|
||||
Invalid slots or base experts map to -1. Shared weights accept any
|
||||
nonnegative base expert and use the adapter slot as the weight index.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
@@ -57,8 +56,20 @@ def _fused_virtual_topk_ids_kernel(
|
||||
# k = offs % top_k # not needed directly
|
||||
|
||||
lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0)
|
||||
mask_val = lora_id >= 0
|
||||
safe_lora = tl.maximum(lora_id, 0)
|
||||
mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS)
|
||||
if HAS_ADAPTER_ENABLED:
|
||||
if NUM_SLOTS == 1:
|
||||
mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0)
|
||||
else:
|
||||
mask_val = mask_val & (
|
||||
tl.load(
|
||||
adapter_enabled_ptr + lora_id * stride_enabled,
|
||||
mask=valid & mask_val,
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
safe_lora = tl.where(mask_val, lora_id, 0)
|
||||
|
||||
base = tl.load(topk_ids_ptr + offs, mask=valid, other=0)
|
||||
if EP_LOCAL:
|
||||
@@ -70,13 +81,13 @@ def _fused_virtual_topk_ids_kernel(
|
||||
base < local_expert_offset + local_num_experts
|
||||
)
|
||||
base = tl.where(owned, base, -1)
|
||||
# Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after
|
||||
# EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on
|
||||
# a real virtual-expert slot belonging to another adapter and trigger OOB
|
||||
# loads in downstream LoRA kernels.
|
||||
shifted = base + safe_lora * num_experts_for_weight
|
||||
result = tl.where(base < 0, base, shifted)
|
||||
result = tl.where(mask_val, result, -1)
|
||||
valid_base = base >= 0
|
||||
if not SHARED_OUTER:
|
||||
valid_base = valid_base & (base < num_experts_for_weight)
|
||||
expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0)
|
||||
shifted = expert + safe_lora * num_experts_for_weight
|
||||
# All align providers use one sentinel bucket; borrowed route IDs stay intact.
|
||||
result = tl.where(mask_val & valid_base, shifted, -1)
|
||||
tl.store(virtual_topk_ids_ptr + offs, result, mask=valid)
|
||||
|
||||
# Write mask once per row (at first k position)
|
||||
@@ -97,48 +108,52 @@ def _fused_virtual_topk_ids(
|
||||
max_loras: int,
|
||||
local_expert_offset: int = 0,
|
||||
local_num_experts: int | None = None,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
"""
|
||||
Returns virtual topk_ids, token_lora_mask, and virtual_num_experts.
|
||||
|
||||
`local_num_experts` (< `num_experts`) enables the EP mask on the per-expert
|
||||
path: non-owned `topk_ids` slots are dropped to the -1 sentinel in-kernel.
|
||||
The shared-outer path routes by lora id, not expert, so it is never masked
|
||||
(else non-owned-but-valid tokens would be dropped on ranks > 0).
|
||||
The shared-outer path drops negative routes but does not apply the local
|
||||
expert range (its weights are shared by every owned expert).
|
||||
"""
|
||||
M, top_k = topk_ids.shape
|
||||
device = topk_ids.device
|
||||
|
||||
if shared_outer:
|
||||
num_experts_for_weight = 1
|
||||
# For shared_outer, we need topk_ids to be zeros
|
||||
zero_topk = torch.zeros_like(topk_ids)
|
||||
input_topk = zero_topk
|
||||
ep_local = False
|
||||
else:
|
||||
num_experts_for_weight = num_experts
|
||||
input_topk = topk_ids
|
||||
ep_local = local_num_experts is not None and local_num_experts < num_experts
|
||||
num_experts_for_weight = 1 if shared_outer else num_experts
|
||||
ep_local = (
|
||||
not shared_outer
|
||||
and local_num_experts is not None
|
||||
and local_num_experts < num_experts
|
||||
)
|
||||
|
||||
virtual_topk_ids = torch.empty_like(topk_ids)
|
||||
token_lora_mask = torch.empty(M, dtype=torch.bool, device=device)
|
||||
if M * top_k == 0:
|
||||
return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,)
|
||||
|
||||
enable_pdl, pdl_kwargs = get_pdl_launch_metadata()
|
||||
_fused_virtual_topk_ids_kernel[grid](
|
||||
input_topk,
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
virtual_topk_ids,
|
||||
token_lora_mask,
|
||||
adapter_enabled,
|
||||
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
|
||||
num_experts_for_weight,
|
||||
shared_outer,
|
||||
M,
|
||||
top_k,
|
||||
BLOCK_SIZE,
|
||||
local_expert_offset,
|
||||
local_num_experts if local_num_experts is not None else 0,
|
||||
ep_local,
|
||||
NUM_SLOTS=max_loras,
|
||||
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
|
||||
ENABLE_PDL=enable_pdl,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
@@ -147,61 +162,6 @@ def _fused_virtual_topk_ids(
|
||||
return virtual_topk_ids, token_lora_mask, virtual_num_experts
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_sanitize_expert_ids_kernel(
|
||||
expert_ids_ptr,
|
||||
output_ptr,
|
||||
num_virtual_experts,
|
||||
N,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
ENABLE_PDL: tl.constexpr = False,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
valid = offs < N
|
||||
|
||||
# GDC wait: expert_ids comes from the immediately preceding align kernel;
|
||||
# wait before consuming it.
|
||||
if ENABLE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0)
|
||||
result = tl.where(eid < num_virtual_experts, eid, -1)
|
||||
tl.store(output_ptr + offs, result, mask=valid)
|
||||
|
||||
# All work is done; hint the runtime to launch the dependent kernel.
|
||||
if ENABLE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
def fused_sanitize_expert_ids(
|
||||
expert_ids: torch.Tensor,
|
||||
num_virtual_experts: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Sanitize expert_ids by replacing values >= num_virtual_experts with -1.
|
||||
|
||||
Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1.
|
||||
"""
|
||||
N = expert_ids.numel()
|
||||
output = torch.empty_like(expert_ids)
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,)
|
||||
|
||||
enable_pdl, pdl_kwargs = get_pdl_launch_metadata()
|
||||
_fused_sanitize_expert_ids_kernel[grid](
|
||||
expert_ids,
|
||||
output,
|
||||
num_virtual_experts,
|
||||
N,
|
||||
BLOCK_SIZE,
|
||||
ENABLE_PDL=enable_pdl,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _moe_lora_shrink_splitk_kernel(
|
||||
# Pointers
|
||||
@@ -238,8 +198,8 @@ def _moe_lora_shrink_splitk_kernel(
|
||||
pid_mn = pid // SPLIT_K
|
||||
|
||||
# GDC wait: the routing buffers (sorted_token_ids/expert_ids/
|
||||
# num_tokens_post_padded) come from the immediately preceding align and
|
||||
# sanitize kernels; wait before consuming them.
|
||||
# num_tokens_post_padded) come from routing preparation; wait before
|
||||
# consuming them.
|
||||
if ENABLE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
@@ -247,6 +207,9 @@ def _moe_lora_shrink_splitk_kernel(
|
||||
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
|
||||
if pid_mn >= num_pid_m * num_pid_n:
|
||||
return
|
||||
|
||||
num_pid_in_group = GROUP_SIZE_M * num_pid_n
|
||||
group_id = pid_mn // num_pid_in_group
|
||||
first_pid_m = group_id * GROUP_SIZE_M
|
||||
@@ -254,9 +217,6 @@ def _moe_lora_shrink_splitk_kernel(
|
||||
pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m)
|
||||
pid_n = (pid_mn % num_pid_in_group) // group_size_m
|
||||
|
||||
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
|
||||
return
|
||||
|
||||
# Token routing (same pattern as fused_moe_triton_kernels)
|
||||
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
|
||||
@@ -417,6 +377,180 @@ def _merged_experts_fused_moe_lora_add_fake(
|
||||
return
|
||||
|
||||
|
||||
def _get_stage_config(
|
||||
weight_shape: tuple[int, ...],
|
||||
stage_top_k: int,
|
||||
dtype: torch.dtype,
|
||||
num_tokens: int,
|
||||
) -> dict[str, Any]:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import (
|
||||
get_config_dtype_str,
|
||||
try_get_optimal_moe_config,
|
||||
)
|
||||
|
||||
config_dtype = get_config_dtype_str(dtype=dtype)
|
||||
get_config_func = functools.partial(
|
||||
try_get_optimal_moe_config,
|
||||
weight_shape,
|
||||
weight_shape,
|
||||
stage_top_k,
|
||||
config_dtype,
|
||||
)
|
||||
try:
|
||||
cfg = get_config_func(num_tokens)
|
||||
except ValueError:
|
||||
K_dim = weight_shape[2]
|
||||
N_dim = weight_shape[1]
|
||||
if K_dim >= 1024:
|
||||
default_block_k = 256
|
||||
elif K_dim >= 64:
|
||||
default_block_k = 64
|
||||
else:
|
||||
default_block_k = max(16, K_dim)
|
||||
cfg = {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": min(64, max(16, N_dim)),
|
||||
"BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)),
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
}
|
||||
return cfg
|
||||
|
||||
|
||||
def _get_shrink_stage_config(
|
||||
weight_shape: tuple[int, ...], M: int, reused_block_size: int | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Choose the shrink kernel config and the matching routing block size."""
|
||||
N = weight_shape[1]
|
||||
if M < 512:
|
||||
config = {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4 if (M <= 4 or N >= 32) else 2,
|
||||
"num_stages": 3,
|
||||
}
|
||||
else:
|
||||
config = {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2}
|
||||
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
|
||||
config = {"BLOCK_SIZE_M": 16, "num_warps": 4, "num_stages": 4}
|
||||
if reused_block_size is not None:
|
||||
config["BLOCK_SIZE_M"] = reused_block_size
|
||||
return config
|
||||
|
||||
|
||||
def _routing_cache_key(
|
||||
num_experts, shared_outer, block_size, include_inactive_routes, adapter_enabled
|
||||
):
|
||||
return (
|
||||
num_experts,
|
||||
shared_outer,
|
||||
block_size,
|
||||
include_inactive_routes,
|
||||
id(adapter_enabled),
|
||||
)
|
||||
|
||||
|
||||
def _align_virtual_topk_ids(
|
||||
virtual_topk_ids, block_size, virtual_num_experts, populated_buckets
|
||||
):
|
||||
if virtual_num_experts < 1024:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
|
||||
moe_align_block_size as native_moe_align_block_size,
|
||||
)
|
||||
|
||||
sorted_ids, expert_ids, count = native_moe_align_block_size(
|
||||
virtual_topk_ids, block_size, virtual_num_experts
|
||||
)
|
||||
else:
|
||||
sorted_ids, expert_ids, count = _align_block_size_large(
|
||||
virtual_topk_ids, block_size, virtual_num_experts
|
||||
)
|
||||
num_tokens = virtual_topk_ids.numel()
|
||||
max_nonempty = min(num_tokens, populated_buckets)
|
||||
tight_padded = (
|
||||
triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size)
|
||||
* block_size
|
||||
)
|
||||
return sorted_ids[:tight_padded], expert_ids[: tight_padded // block_size], count
|
||||
|
||||
|
||||
def _prewarm_per_expert_lora_routes(
|
||||
hidden_states: torch.Tensor,
|
||||
lora_a: torch.Tensor,
|
||||
lora_b: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
routing_cache: dict,
|
||||
*,
|
||||
shrink_lora_b: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
include_inactive_routes: bool = False,
|
||||
local_expert_offset: int = 0,
|
||||
local_num_experts: int | None = None,
|
||||
) -> bool:
|
||||
"""Prepare paired per-expert stages on main and release raw IDs before returning.
|
||||
|
||||
The aligned cache retains only tables and their shared token mask. Global-ID
|
||||
EP routing keeps the existing provider admission and staged fallback.
|
||||
"""
|
||||
if lora_a.shape[:2] != lora_b.shape[:2]:
|
||||
return False
|
||||
slots, num_experts = lora_a.shape[:2]
|
||||
if local_num_experts is not None and local_num_experts < num_experts:
|
||||
return False
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (slots,)
|
||||
assert adapter_enabled.dtype == torch.int32
|
||||
assert adapter_enabled.device == topk_ids.device
|
||||
if topk_ids.numel() == 0:
|
||||
return True
|
||||
a_shape = (slots * num_experts, lora_a.shape[2], lora_a.shape[3])
|
||||
b_shape = (slots * num_experts, lora_b.shape[2], lora_b.shape[3])
|
||||
tokens = token_lora_mapping.shape[0]
|
||||
b_config = _get_stage_config(b_shape, 1, hidden_states.dtype, tokens)
|
||||
reuse_b_block = (
|
||||
tokens >= 512 and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
|
||||
)
|
||||
reused_block_size = None
|
||||
if reuse_b_block:
|
||||
shrink_b_shape = (
|
||||
shrink_lora_b.shape[0] * shrink_lora_b.shape[1],
|
||||
shrink_lora_b.shape[2],
|
||||
shrink_lora_b.shape[3],
|
||||
)
|
||||
reused_block_size = _get_stage_config(
|
||||
shrink_b_shape, 1, hidden_states.dtype, tokens
|
||||
)["BLOCK_SIZE_M"]
|
||||
a_config = _get_shrink_stage_config(a_shape, tokens, reused_block_size)
|
||||
missing = {}
|
||||
for block_size in (b_config["BLOCK_SIZE_M"], a_config["BLOCK_SIZE_M"]):
|
||||
key = _routing_cache_key(
|
||||
num_experts, False, block_size, include_inactive_routes, adapter_enabled
|
||||
)
|
||||
if key not in routing_cache:
|
||||
missing[key] = block_size
|
||||
if not missing:
|
||||
return True
|
||||
virtual_ids, token_mask, virtual_experts = _fused_virtual_topk_ids(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
num_experts,
|
||||
False,
|
||||
slots,
|
||||
local_expert_offset,
|
||||
local_num_experts,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
for key, block_size in missing.items():
|
||||
sorted_ids, expert_ids, count = _align_virtual_topk_ids(
|
||||
virtual_ids, block_size, virtual_experts, virtual_experts
|
||||
)
|
||||
routing_cache[key] = (sorted_ids, expert_ids, count, token_mask)
|
||||
# Only aligned tables/masks survive the main-stream preparation scope.
|
||||
del virtual_ids
|
||||
return True
|
||||
|
||||
|
||||
def _merged_experts_fused_moe_lora_add_impl(
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -441,6 +575,8 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
prewarm_a_routing: bool = True,
|
||||
prewarm_b_routing: bool = True,
|
||||
zero_intermediate: bool = False,
|
||||
include_inactive_routes: bool = False,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> "torch.Tensor | None":
|
||||
"""
|
||||
1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts.
|
||||
@@ -462,6 +598,10 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
``broadcast_intermediate`` is an expand-only mode where one rank vector per
|
||||
token is reused for every routed expert.
|
||||
|
||||
``include_inactive_routes`` retains negative routing blocks. The existing
|
||||
expand then writes literal zero to private overwrite outputs, while ADD
|
||||
projections preserve their inactive rows. Default routing is unchanged.
|
||||
|
||||
EP accepts either global weights/IDs with a local range or already-localized
|
||||
weights/IDs from the standard dispatcher.
|
||||
"""
|
||||
@@ -470,82 +610,17 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
# dim 1) while lora_b is per-expert, so take the max for the true global count.
|
||||
per_expert_dim = max(lora_a.shape[1], lora_b.shape[1])
|
||||
ep_local = local_num_experts is not None and local_num_experts < per_expert_dim
|
||||
input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1]
|
||||
num_routes = topk_ids.numel()
|
||||
input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1]
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (max_loras,)
|
||||
assert adapter_enabled.dtype == torch.int32
|
||||
assert adapter_enabled.device == topk_ids.device
|
||||
|
||||
def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor:
|
||||
# [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y]
|
||||
return t.reshape(t.shape[0] * t.shape[1], t.shape[2], t.shape[3])
|
||||
|
||||
def _get_stage_config(
|
||||
weight: torch.Tensor,
|
||||
stage_top_k: int,
|
||||
) -> dict[str, Any]:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import (
|
||||
get_config_dtype_str,
|
||||
try_get_optimal_moe_config,
|
||||
)
|
||||
|
||||
config_dtype = get_config_dtype_str(dtype=hidden_states.dtype)
|
||||
get_config_func = functools.partial(
|
||||
try_get_optimal_moe_config,
|
||||
weight.shape,
|
||||
weight.shape,
|
||||
stage_top_k,
|
||||
config_dtype,
|
||||
)
|
||||
try:
|
||||
cfg = get_config_func(token_lora_mapping.shape[0])
|
||||
except ValueError:
|
||||
K_dim = weight.shape[2]
|
||||
N_dim = weight.shape[1]
|
||||
if K_dim >= 1024:
|
||||
default_block_k = 256
|
||||
elif K_dim >= 64:
|
||||
default_block_k = 64
|
||||
else:
|
||||
default_block_k = max(16, K_dim)
|
||||
cfg = {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": min(64, max(16, N_dim)),
|
||||
"BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)),
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
}
|
||||
return cfg
|
||||
|
||||
def _get_shrink_stage_config(weight: torch.Tensor, M: int) -> dict[str, Any]:
|
||||
"""Heuristic launch config for the LoRA A (shrink) stage.
|
||||
|
||||
Only BLOCK_SIZE_M / num_warps / num_stages are chosen here (a simple
|
||||
decode/prefill rule). SPLIT_K is intentionally NOT set: the launcher
|
||||
derives it from the post-alignment grid, see _invoke_moe_lora_shrink_splitk.
|
||||
M is the number of input tokens; N (= weight.shape[1]) is the LoRA rank.
|
||||
"""
|
||||
N = weight.shape[1]
|
||||
if M < 512: # decode / small batch: pin a small block, latency-bound
|
||||
return {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4 if (M <= 4 or N >= 32) else 2,
|
||||
"num_stages": 3,
|
||||
}
|
||||
return {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2} # prefill
|
||||
|
||||
def _align_block_size(
|
||||
topk_ids: torch.Tensor,
|
||||
block_size: int,
|
||||
num_experts: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# The native align kernel consumes num_experts + 1 internally for its
|
||||
# sentinel bucket, so the 1024-expert boundary must use the fallback path.
|
||||
if num_experts < 1024:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
|
||||
moe_align_block_size as native_moe_align_block_size,
|
||||
)
|
||||
|
||||
return native_moe_align_block_size(topk_ids, block_size, num_experts)
|
||||
return _align_block_size_large(topk_ids, block_size, num_experts)
|
||||
|
||||
def _get_routing(
|
||||
topk_ids: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
@@ -554,7 +629,13 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
block_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Check routing_cache for cross-call reuse (gate_up and down share routing)
|
||||
cache_key = (num_experts, shared_outer, block_size)
|
||||
cache_key = _routing_cache_key(
|
||||
num_experts,
|
||||
shared_outer,
|
||||
block_size,
|
||||
include_inactive_routes,
|
||||
adapter_enabled,
|
||||
)
|
||||
if routing_cache is not None:
|
||||
cached = routing_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
@@ -593,8 +674,9 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
block_size,
|
||||
local_expert_offset,
|
||||
local_num_experts,
|
||||
do_skip=True,
|
||||
do_skip=not include_inactive_routes,
|
||||
compact=compact_merged,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
result = (
|
||||
sorted_token_ids,
|
||||
@@ -615,36 +697,16 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
max_loras,
|
||||
local_expert_offset,
|
||||
local_num_experts,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
)
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size(
|
||||
virtual_topk_ids,
|
||||
block_size=block_size,
|
||||
num_experts=virtual_num_experts,
|
||||
)
|
||||
# _align_block_size uses a worst-case padded allocation. Trim the routing buffers
|
||||
# to a tighter upper bound so we keep the real routed work but drop unused padding.
|
||||
# Under EP only this rank's experts are populated, so the per-expert stage bounds
|
||||
# the non-empty buckets by `local_num_experts * max_loras` owned (expert, lora)
|
||||
# combos + 1 sentinel bucket (all non-owned -> -1). The +1 matters: each non-empty
|
||||
# bucket adds up to block-1 padding, so omitting it could truncate real tokens.
|
||||
num_tokens = topk_ids.numel()
|
||||
if ep_local and not shared_outer and local_num_experts < num_experts:
|
||||
populated_buckets = local_num_experts * max_loras + 1
|
||||
else:
|
||||
populated_buckets = virtual_num_experts
|
||||
max_nonempty = min(num_tokens, populated_buckets)
|
||||
tight_padded = (
|
||||
triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size)
|
||||
* block_size
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_virtual_topk_ids(
|
||||
virtual_topk_ids, block_size, virtual_num_experts, populated_buckets
|
||||
)
|
||||
sorted_token_ids = sorted_token_ids[:tight_padded]
|
||||
expert_ids = expert_ids[: tight_padded // block_size]
|
||||
# Single adapter (max_loras == 1): the per-adapter virtual-expert offset
|
||||
# (safe_lora * num_experts) is always 0, so align never emits an id >=
|
||||
# virtual_num_experts and fused_sanitize_expert_ids is an identity -> skip it.
|
||||
if max_loras != 1:
|
||||
expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts)
|
||||
result = (
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -673,11 +735,42 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
assert intermediate_buffer is not None
|
||||
assert intermediate_buffer.ndim == 2
|
||||
assert intermediate_buffer.shape[0] == token_lora_mapping.shape[0]
|
||||
lora_a_virtual = _merge_lora_expert_weight(lora_a)
|
||||
lora_b_virtual = _merge_lora_expert_weight(lora_b)
|
||||
if num_routes == 0:
|
||||
if stage == "shrink":
|
||||
if intermediate_buffer is not None:
|
||||
return intermediate_buffer
|
||||
return hidden_states.new_empty(
|
||||
(token_lora_mapping.shape[0], topk_ids.shape[1], max_lora_rank)
|
||||
)
|
||||
if stage == "expand":
|
||||
assert intermediate_buffer is not None, (
|
||||
"stage='expand' requires intermediate_buffer"
|
||||
)
|
||||
return None
|
||||
|
||||
num_experts_a = lora_a.shape[1]
|
||||
num_experts_b = lora_b.shape[1]
|
||||
b_stage_config = _get_stage_config(lora_b_virtual, 1)
|
||||
a_shape = (max_loras * num_experts_a, lora_a.shape[2], lora_a.shape[3])
|
||||
b_shape = (lora_b.shape[0] * num_experts_b, lora_b.shape[2], lora_b.shape[3])
|
||||
tokens = token_lora_mapping.shape[0]
|
||||
needs_shrink_config = stage in ("all", "shrink") or (
|
||||
stage == "routing" and prewarm_a_routing
|
||||
)
|
||||
reuse_b_block = (
|
||||
needs_shrink_config
|
||||
and tokens >= 512
|
||||
and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
|
||||
)
|
||||
needs_b_config = (
|
||||
stage in ("all", "expand")
|
||||
or (prewarm_b_routing and (stage == "routing" or routing_cache is not None))
|
||||
or reuse_b_block
|
||||
)
|
||||
b_stage_config = (
|
||||
_get_stage_config(b_shape, 1, hidden_states.dtype, tokens)
|
||||
if needs_b_config
|
||||
else None
|
||||
)
|
||||
|
||||
if stage == "routing":
|
||||
# Pre-warm the routing cache on the CALLER'S (main) stream so the
|
||||
@@ -687,20 +780,12 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
# allocator's stream tracking is disabled while capturing), corrupting
|
||||
# replays. Routing needs only topk_ids + token_lora_mapping, which are
|
||||
# both ready before the side-stream fork, so it can run on main.
|
||||
a_cfg = _get_shrink_stage_config(lora_a_virtual, token_lora_mapping.shape[0])
|
||||
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
|
||||
a_cfg = {**a_cfg, "BLOCK_SIZE_M": 16}
|
||||
# Match the actual shrink-stage override below. Without this, callers
|
||||
# that admit prefill-shaped batches into a side stream prewarm block-32
|
||||
# routing here, then miss the cache when shrink switches to the B-stage
|
||||
# block size. The miss allocates routing buffers on the side stream
|
||||
# during capture, violating the allocation guarantee of stage='routing'.
|
||||
if (
|
||||
lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
|
||||
and token_lora_mapping.shape[0] >= 512
|
||||
):
|
||||
a_cfg["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"]
|
||||
if prewarm_a_routing:
|
||||
a_cfg = _get_shrink_stage_config(
|
||||
a_shape,
|
||||
tokens,
|
||||
b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None,
|
||||
)
|
||||
_get_routing(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
@@ -709,6 +794,7 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
a_cfg["BLOCK_SIZE_M"],
|
||||
)
|
||||
if prewarm_b_routing:
|
||||
assert b_stage_config is not None
|
||||
_get_routing(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
@@ -721,22 +807,9 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
intermediate = intermediate_buffer
|
||||
if stage != "expand":
|
||||
a_stage_config = _get_shrink_stage_config(
|
||||
lora_a_virtual, token_lora_mapping.shape[0]
|
||||
a_shape, tokens, b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None
|
||||
)
|
||||
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
|
||||
# Test-only override; the launcher fixes N and K block sizes.
|
||||
a_stage_config = {
|
||||
**a_stage_config,
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
}
|
||||
# Match routing block sizes so prefill stages can share cached alignment.
|
||||
if (
|
||||
lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
|
||||
and token_lora_mapping.shape[0] >= 512
|
||||
):
|
||||
a_stage_config["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"]
|
||||
lora_a_virtual = _merge_lora_expert_weight(lora_a)
|
||||
(
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -800,6 +873,7 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
# Pre-warm the routing-B cache on this (side) stream so the later "expand" stage
|
||||
# launches no routing kernels — they overlap finalize together with the shrink.
|
||||
if routing_cache is not None and prewarm_b_routing:
|
||||
assert b_stage_config is not None
|
||||
_get_routing(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
@@ -810,6 +884,8 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
return intermediate
|
||||
|
||||
assert intermediate is not None, "stage='expand' requires intermediate_buffer"
|
||||
assert b_stage_config is not None
|
||||
lora_b_virtual = _merge_lora_expert_weight(lora_b)
|
||||
(
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -946,6 +1022,8 @@ def merged_experts_fused_moe_lora_add(
|
||||
prewarm_a_routing: bool = True,
|
||||
prewarm_b_routing: bool = True,
|
||||
zero_intermediate: bool = False,
|
||||
include_inactive_routes: bool = False,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> "torch.Tensor | None":
|
||||
"""Public API: wraps the registered op with routing_cache support."""
|
||||
return _merged_experts_fused_moe_lora_add_impl(
|
||||
@@ -972,4 +1050,6 @@ def merged_experts_fused_moe_lora_add(
|
||||
prewarm_a_routing=prewarm_a_routing,
|
||||
prewarm_b_routing=prewarm_b_routing,
|
||||
zero_intermediate=zero_intermediate,
|
||||
include_inactive_routes=include_inactive_routes,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
|
||||
@@ -20,22 +20,21 @@ def _fused_virtual_topk_ids_kernel(
|
||||
token_lora_mapping_ptr,
|
||||
virtual_topk_ids_ptr,
|
||||
token_lora_mask_ptr,
|
||||
adapter_enabled_ptr,
|
||||
stride_enabled,
|
||||
num_experts_for_weight: tl.constexpr,
|
||||
SHARED_OUTER: tl.constexpr,
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
HAS_ADAPTER_ENABLED: tl.constexpr,
|
||||
M,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel.
|
||||
Map active slots and valid base experts to virtual weight indices.
|
||||
|
||||
For each (m, k):
|
||||
lora_id = token_lora_mapping[m]
|
||||
mask[m] = (lora_id >= 0)
|
||||
safe_lora = max(lora_id, 0)
|
||||
if shared_outer: (handled by num_experts_for_weight == 0 sentinel)
|
||||
virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora)
|
||||
else:
|
||||
virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight
|
||||
Invalid slots or base experts map to -1. Shared weights accept any
|
||||
nonnegative base expert and use the adapter slot as the weight index.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
@@ -46,16 +45,29 @@ def _fused_virtual_topk_ids_kernel(
|
||||
# k = offs % top_k # not needed directly
|
||||
|
||||
lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0)
|
||||
mask_val = lora_id >= 0
|
||||
safe_lora = tl.maximum(lora_id, 0)
|
||||
mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS)
|
||||
if HAS_ADAPTER_ENABLED:
|
||||
if NUM_SLOTS == 1:
|
||||
mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0)
|
||||
else:
|
||||
mask_val = mask_val & (
|
||||
tl.load(
|
||||
adapter_enabled_ptr + lora_id * stride_enabled,
|
||||
mask=valid & mask_val,
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
safe_lora = tl.where(mask_val, lora_id, 0)
|
||||
|
||||
base = tl.load(topk_ids_ptr + offs, mask=valid, other=0)
|
||||
# Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after
|
||||
# EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on
|
||||
# a real virtual-expert slot belonging to another adapter and trigger OOB
|
||||
# loads in downstream LoRA kernels.
|
||||
shifted = base + safe_lora * num_experts_for_weight
|
||||
result = tl.where(base < 0, base, shifted)
|
||||
valid_base = base >= 0
|
||||
if not SHARED_OUTER:
|
||||
valid_base = valid_base & (base < num_experts_for_weight)
|
||||
expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0)
|
||||
shifted = expert + safe_lora * num_experts_for_weight
|
||||
# All align providers use one sentinel bucket; borrowed route IDs stay intact.
|
||||
result = tl.where(mask_val & valid_base, shifted, -1)
|
||||
tl.store(virtual_topk_ids_ptr + offs, result, mask=valid)
|
||||
|
||||
# Write mask once per row (at first k position)
|
||||
@@ -70,6 +82,7 @@ def _fused_virtual_topk_ids(
|
||||
num_experts: int,
|
||||
shared_outer: bool,
|
||||
max_loras: int,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
"""
|
||||
Returns virtual topk_ids, token_lora_mask, and virtual_num_experts.
|
||||
@@ -77,27 +90,27 @@ def _fused_virtual_topk_ids(
|
||||
M, top_k = topk_ids.shape
|
||||
device = topk_ids.device
|
||||
|
||||
if shared_outer:
|
||||
num_experts_for_weight = 1
|
||||
# For shared_outer, we need topk_ids to be zeros
|
||||
zero_topk = torch.zeros_like(topk_ids)
|
||||
input_topk = zero_topk
|
||||
else:
|
||||
num_experts_for_weight = num_experts
|
||||
input_topk = topk_ids
|
||||
num_experts_for_weight = 1 if shared_outer else num_experts
|
||||
|
||||
virtual_topk_ids = torch.empty_like(topk_ids)
|
||||
token_lora_mask = torch.empty(M, dtype=torch.bool, device=device)
|
||||
if M * top_k == 0:
|
||||
return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,)
|
||||
|
||||
_fused_virtual_topk_ids_kernel[grid](
|
||||
input_topk,
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
virtual_topk_ids,
|
||||
token_lora_mask,
|
||||
adapter_enabled,
|
||||
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
|
||||
num_experts_for_weight,
|
||||
shared_outer,
|
||||
max_loras,
|
||||
adapter_enabled is not None,
|
||||
M,
|
||||
top_k,
|
||||
BLOCK_SIZE,
|
||||
@@ -107,48 +120,6 @@ def _fused_virtual_topk_ids(
|
||||
return virtual_topk_ids, token_lora_mask, virtual_num_experts
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_sanitize_expert_ids_kernel(
|
||||
expert_ids_ptr,
|
||||
output_ptr,
|
||||
num_virtual_experts,
|
||||
N,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
valid = offs < N
|
||||
|
||||
eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0)
|
||||
result = tl.where(eid < num_virtual_experts, eid, -1)
|
||||
tl.store(output_ptr + offs, result, mask=valid)
|
||||
|
||||
|
||||
def fused_sanitize_expert_ids(
|
||||
expert_ids: torch.Tensor,
|
||||
num_virtual_experts: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Sanitize expert_ids by replacing values >= num_virtual_experts with -1.
|
||||
|
||||
Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1.
|
||||
"""
|
||||
N = expert_ids.numel()
|
||||
output = torch.empty_like(expert_ids)
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,)
|
||||
|
||||
_fused_sanitize_expert_ids_kernel[grid](
|
||||
expert_ids,
|
||||
output,
|
||||
num_virtual_experts,
|
||||
N,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _moe_lora_shrink_splitk_kernel(
|
||||
# Pointers
|
||||
@@ -187,6 +158,9 @@ def _moe_lora_shrink_splitk_kernel(
|
||||
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
|
||||
if pid_mn >= num_pid_m * num_pid_n:
|
||||
return
|
||||
|
||||
num_pid_in_group = GROUP_SIZE_M * num_pid_n
|
||||
group_id = pid_mn // num_pid_in_group
|
||||
first_pid_m = group_id * GROUP_SIZE_M
|
||||
@@ -194,9 +168,6 @@ def _moe_lora_shrink_splitk_kernel(
|
||||
pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m)
|
||||
pid_n = (pid_mn % num_pid_in_group) // group_size_m
|
||||
|
||||
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
|
||||
return
|
||||
|
||||
# Token routing (same pattern as fused_moe_triton_kernels)
|
||||
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
|
||||
@@ -309,7 +280,7 @@ def _align_block_size_jit(
|
||||
scratch=None,
|
||||
*,
|
||||
return_inactive_prefix_count: bool = False,
|
||||
map_invalid_experts_to_sentinel: bool = False,
|
||||
map_invalid_experts_to_sentinel: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""CUDA JIT align_block_size for num_experts > 1024 (up to 8191).
|
||||
|
||||
@@ -319,8 +290,8 @@ def _align_block_size_jit(
|
||||
|
||||
``scratch(numel)`` returns the int32 scratch buffer (default: a fresh
|
||||
allocation); a caller under CUDA graphs passes a persistent buffer.
|
||||
IDs must be certified in [-1, num_experts) unless
|
||||
``map_invalid_experts_to_sentinel`` maps other signed IDs to bucket zero.
|
||||
Out-of-range signed IDs share the sentinel bucket; opt out only for IDs
|
||||
certified in [-1, num_experts).
|
||||
"""
|
||||
assert num_experts <= 8191, (
|
||||
f"_align_block_size_jit supports at most 8191 experts "
|
||||
@@ -544,6 +515,9 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
experts_shared_outer_loras_a: bool,
|
||||
experts_shared_outer_loras_b: bool,
|
||||
routing_cache: dict | None = None,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
*,
|
||||
a_output_slices: int = 1,
|
||||
) -> None:
|
||||
"""
|
||||
1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts.
|
||||
@@ -551,8 +525,21 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
3. Run regular SGLang fused-MoE kernels for LoRA A and LoRA B.
|
||||
4. Mask out tokens with token_lora_mapping == -1 on the add path.
|
||||
"""
|
||||
assert a_output_slices in (1, 2)
|
||||
if a_output_slices == 2:
|
||||
assert lora_a.shape[2] == 2 * lora_b.shape[3]
|
||||
assert lora_b.shape[2] > 0 and lora_b.shape[2] % 2 == 0
|
||||
assert output.shape[-1] == lora_b.shape[2]
|
||||
max_loras, _, max_lora_rank, _ = lora_a.shape
|
||||
input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1]
|
||||
num_routes = topk_ids.numel()
|
||||
input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1]
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (max_loras,)
|
||||
assert adapter_enabled.device == topk_ids.device
|
||||
assert adapter_enabled.dtype == torch.int32
|
||||
|
||||
if num_routes == 0:
|
||||
return
|
||||
|
||||
def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor:
|
||||
# [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y]
|
||||
@@ -619,7 +606,7 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
block_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Check routing_cache for cross-call reuse (gate_up and down share routing)
|
||||
cache_key = (num_experts, shared_outer, block_size)
|
||||
cache_key = (num_experts, shared_outer, block_size, id(adapter_enabled))
|
||||
if routing_cache is not None:
|
||||
cached = routing_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
@@ -627,7 +614,12 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
|
||||
virtual_topk_ids, token_lora_mask, virtual_num_experts = (
|
||||
_fused_virtual_topk_ids(
|
||||
topk_ids, token_lora_mapping, num_experts, shared_outer, max_loras
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
num_experts,
|
||||
shared_outer,
|
||||
max_loras,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
)
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size(
|
||||
@@ -645,7 +637,6 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
)
|
||||
sorted_token_ids = sorted_token_ids[:tight_padded]
|
||||
expert_ids = expert_ids[: tight_padded // block_size]
|
||||
expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts)
|
||||
result = (
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -739,6 +730,7 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
fuse_add_to_output=True,
|
||||
add_output_mask=token_lora_mask,
|
||||
router_topk=topk_ids.shape[1],
|
||||
a_output_slices=a_output_slices,
|
||||
)
|
||||
|
||||
|
||||
@@ -790,8 +782,11 @@ def merged_experts_fused_moe_lora_add(
|
||||
experts_shared_outer_loras_a: bool,
|
||||
experts_shared_outer_loras_b: bool,
|
||||
routing_cache: dict | None = None,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
*,
|
||||
a_output_slices: int = 1,
|
||||
) -> None:
|
||||
"""Public API: wraps the registered op with routing_cache support."""
|
||||
"""Add virtual-expert LoRA deltas, optionally pairing A rank/output halves."""
|
||||
_merged_experts_fused_moe_lora_add_impl(
|
||||
output,
|
||||
hidden_states,
|
||||
@@ -804,4 +799,6 @@ def merged_experts_fused_moe_lora_add(
|
||||
experts_shared_outer_loras_a,
|
||||
experts_shared_outer_loras_b,
|
||||
routing_cache,
|
||||
adapter_enabled=adapter_enabled,
|
||||
a_output_slices=a_output_slices,
|
||||
)
|
||||
|
||||
@@ -382,6 +382,9 @@ def fused_moe_kernel(
|
||||
FUSE_SWIGLU: tl.constexpr = False,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
GDC_EARLY: tl.constexpr = False,
|
||||
A_OUTPUT_SLICES: tl.constexpr = 1,
|
||||
A_OUTPUT_SLICE_WIDTH: tl.constexpr = 0,
|
||||
A_OUTPUT_SLICE_K: tl.constexpr = 0,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
@@ -420,7 +423,14 @@ def fused_moe_kernel(
|
||||
# This is done in a grouped ordering to promote L2 data reuse.
|
||||
pid = tl.program_id(axis=0)
|
||||
num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
if A_OUTPUT_SLICES == 2:
|
||||
# Gate/up outputs use separate rank halves of the same shrink result.
|
||||
# Keep each N tile inside one half, including ragged output widths.
|
||||
slice_width: tl.constexpr = A_OUTPUT_SLICE_WIDTH
|
||||
num_pid_n_per_slice: tl.constexpr = tl.cdiv(slice_width, BLOCK_SIZE_N)
|
||||
num_pid_n = 2 * num_pid_n_per_slice
|
||||
else:
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
num_pid_in_group = GROUP_SIZE_M * num_pid_n
|
||||
group_id = pid // num_pid_in_group
|
||||
first_pid_m = group_id * GROUP_SIZE_M
|
||||
@@ -445,7 +455,11 @@ def fused_moe_kernel(
|
||||
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
|
||||
off_experts = off_experts_i32.to(tl.int64)
|
||||
|
||||
if off_experts == -1:
|
||||
if A_OUTPUT_SLICES == 2:
|
||||
inactive_expert = off_experts < 0
|
||||
else:
|
||||
inactive_expert = off_experts == -1
|
||||
if inactive_expert:
|
||||
# The down GEMM never reads inactive sorted or half-width intermediates.
|
||||
# Add/reduce outputs already hold their initial value and must stay intact.
|
||||
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
|
||||
@@ -463,7 +477,15 @@ def fused_moe_kernel(
|
||||
)
|
||||
return
|
||||
|
||||
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
|
||||
if A_OUTPUT_SLICES == 2:
|
||||
output_slice = pid_n // num_pid_n_per_slice
|
||||
local_pid_n = pid_n % num_pid_n_per_slice
|
||||
local_columns = local_pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(
|
||||
tl.int64
|
||||
)
|
||||
offs_bn = local_columns % slice_width + output_slice * slice_width
|
||||
else:
|
||||
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
|
||||
offs_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
if a_desc is not None:
|
||||
assert use_fp8_w8a8 and group_n > 0 and group_k > 0
|
||||
@@ -472,6 +494,8 @@ def fused_moe_kernel(
|
||||
a_ptrs = a_ptr + (
|
||||
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
|
||||
)
|
||||
if A_OUTPUT_SLICES == 2:
|
||||
a_ptrs += output_slice * A_OUTPUT_SLICE_K * stride_ak
|
||||
|
||||
if b_desc is not None:
|
||||
start_offs_n = pid_n * BLOCK_SIZE_N
|
||||
@@ -610,7 +634,12 @@ def fused_moe_kernel(
|
||||
accumulator = accumulator.to(compute_type)
|
||||
# -----------------------------------------------------------
|
||||
# Write back the block of the output
|
||||
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
|
||||
if A_OUTPUT_SLICES == 2:
|
||||
offs_cn = local_columns + output_slice * slice_width
|
||||
output_column_mask = local_columns < slice_width
|
||||
else:
|
||||
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
|
||||
output_column_mask = offs_cn < N
|
||||
|
||||
if FUSE_SWIGLU:
|
||||
# W13 rows were interleaved at load time, so gate/up of the same
|
||||
@@ -667,7 +696,7 @@ def fused_moe_kernel(
|
||||
offs_token_out = offs_token // ROUTER_TOPK
|
||||
add_mask = tl.load(add_mask_ptr + offs_token_out, mask=token_mask, other=False)
|
||||
c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :]
|
||||
c_mask = token_mask[:, None] & add_mask[:, None] & (offs_cn[None, :] < N)
|
||||
c_mask = token_mask[:, None] & add_mask[:, None] & output_column_mask[None, :]
|
||||
existing = tl.load(c_ptrs, mask=c_mask, other=0.0)
|
||||
tl.store(c_ptrs, existing + accumulator, mask=c_mask)
|
||||
# ===== TO BE REFACTORED ====
|
||||
@@ -810,10 +839,38 @@ def invoke_fused_moe_kernel(
|
||||
mask_output: bool = False,
|
||||
lora_preserve_base: bool = False,
|
||||
fuse_swiglu: bool = False,
|
||||
a_output_slices: int = 1,
|
||||
) -> None:
|
||||
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
|
||||
"""Skip inactive experts, including with filter_expert=False.
|
||||
|
||||
a_output_slices=2 pairs consecutive A rank halves with B/C output halves
|
||||
on the unquantized add path, without splitting the shrink/expand launches.
|
||||
"""
|
||||
assert topk_weights.stride(1) == 1
|
||||
assert sorted_token_ids.stride(0) == 1
|
||||
assert a_output_slices in (1, 2)
|
||||
a_output_slice_width = 0
|
||||
a_output_slice_k = 0
|
||||
if a_output_slices == 2:
|
||||
assert fuse_add_to_output
|
||||
assert add_output_mask is not None
|
||||
assert not mul_routed_weight
|
||||
assert not (use_fp8_w8a8 or use_int8_w8a8 or use_int8_w8a16 or use_int4_w4a16)
|
||||
assert not per_channel_quant and block_shape is None
|
||||
assert not (a_use_tma or b_use_tma or fuse_swiglu or c_sorted)
|
||||
assert not (
|
||||
fuse_sum_all_reduce or mask_output or lora_preserve_base or no_combine
|
||||
)
|
||||
assert bias is None and A_scale is None and B_scale is None and B_zp is None
|
||||
assert A.dtype in (torch.float16, torch.bfloat16)
|
||||
assert B.dtype == C.dtype == A.dtype
|
||||
assert compute_type == (tl.float16 if A.dtype == torch.float16 else tl.bfloat16)
|
||||
assert A.ndim == 2 and B.ndim == 3
|
||||
assert A.shape[1] == 2 * B.shape[2]
|
||||
assert B.shape[1] > 0 and B.shape[1] % 2 == 0
|
||||
assert C.shape[-1] == B.shape[1]
|
||||
a_output_slice_width = B.shape[1] // 2
|
||||
a_output_slice_k = B.shape[2]
|
||||
|
||||
if fuse_swiglu:
|
||||
# The epilogue assumes an interleaved-gate/up bf16 up-GEMM writing a
|
||||
@@ -887,7 +944,11 @@ def invoke_fused_moe_kernel(
|
||||
|
||||
grid = lambda META: (
|
||||
triton.cdiv(sorted_token_ids.shape[0], META["BLOCK_SIZE_M"])
|
||||
* triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]),
|
||||
* (
|
||||
triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"])
|
||||
if a_output_slices == 1
|
||||
else 2 * triton.cdiv(a_output_slice_width, META["BLOCK_SIZE_N"])
|
||||
),
|
||||
)
|
||||
|
||||
K = B.shape[2] - padded_size
|
||||
@@ -1037,6 +1098,9 @@ def invoke_fused_moe_kernel(
|
||||
c_sorted=c_sorted,
|
||||
swap_ab=swap_ab,
|
||||
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
|
||||
A_OUTPUT_SLICES=a_output_slices,
|
||||
A_OUTPUT_SLICE_WIDTH=a_output_slice_width,
|
||||
A_OUTPUT_SLICE_K=a_output_slice_k,
|
||||
MASK_OUTPUT=mask_output,
|
||||
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
|
||||
ROUTER_TOPK=router_topk,
|
||||
|
||||
@@ -72,7 +72,7 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
topk_weights = topk_output.topk_weights
|
||||
|
||||
assert runner_config.activation == "silu", "Only SiLU activation is supported."
|
||||
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
|
||||
@@ -96,7 +96,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
quant_info.global_num_experts if quant_info.expert_map is not None else E
|
||||
)
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, block_size_m, align_num_experts
|
||||
topk_ids,
|
||||
block_size_m,
|
||||
align_num_experts,
|
||||
return_inactive_prefix_count=quant_info.expert_map is None,
|
||||
)
|
||||
|
||||
# Per-call workspace like fused_experts_none_to_marlin: a shared buffer aliases
|
||||
@@ -119,13 +122,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
)
|
||||
|
||||
# Stage 1: Gate/Up (Marlin)
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -145,6 +144,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
top_k=topk,
|
||||
mul_topk_weights=False,
|
||||
is_ep=quant_info.expert_map is not None,
|
||||
expert_ids_are_sorted=quant_info.expert_map is None,
|
||||
has_inactive_prefix_count=quant_info.expert_map is None,
|
||||
b_q_type=scalar_type1,
|
||||
size_m=M,
|
||||
size_n=2 * N,
|
||||
@@ -169,6 +170,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: Down (Marlin)
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
@@ -188,7 +192,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
moe_block_size=block_size_m,
|
||||
top_k=1,
|
||||
mul_topk_weights=True,
|
||||
zero_inactive_output=True,
|
||||
is_ep=quant_info.expert_map is not None,
|
||||
expert_ids_are_sorted=quant_info.expert_map is None,
|
||||
has_inactive_prefix_count=quant_info.expert_map is None,
|
||||
b_q_type=scalar_type2,
|
||||
size_m=M * topk,
|
||||
size_n=K,
|
||||
|
||||
@@ -189,7 +189,12 @@ class LoRAInfo:
|
||||
|
||||
@dataclass
|
||||
class LoRAHooks:
|
||||
"""Hook callbacks for injecting LoRA deltas into the MoE pipeline."""
|
||||
"""Hook callbacks for injecting LoRA deltas into the MoE pipeline.
|
||||
|
||||
Dropped route IDs remain negative. Hooks must skip those routes before
|
||||
reading their payloads or scores; private inactive intermediates may be
|
||||
uninitialized and routing scores are passed without a sanitized copy.
|
||||
"""
|
||||
|
||||
after_gate_up: (
|
||||
Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], None] | None
|
||||
@@ -357,6 +362,8 @@ def _add_lora_gate_up_delta(
|
||||
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=routing_cache,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
a_output_slices=2 if is_gated else 1,
|
||||
)
|
||||
else:
|
||||
blk = _get_moe_lora_block_config(r)
|
||||
@@ -432,6 +439,7 @@ def _add_lora_down_delta(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
|
||||
routing_cache=routing_cache,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
)
|
||||
else:
|
||||
blk = _get_moe_lora_block_config(lora_info.max_lora_rank)
|
||||
|
||||
@@ -14,11 +14,34 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _silu_mul_add_delta_row(
|
||||
x_ptr,
|
||||
d_ptr,
|
||||
out_ptr,
|
||||
N,
|
||||
row,
|
||||
cols,
|
||||
mask,
|
||||
delta_mask,
|
||||
):
|
||||
x_row = x_ptr + row * 2 * N
|
||||
d_row = d_ptr + row * 2 * N
|
||||
gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
gate += tl.load(d_row + cols, mask=delta_mask, other=0.0).to(tl.float32)
|
||||
up += tl.load(d_row + N + cols, mask=delta_mask, other=0.0).to(tl.float32)
|
||||
|
||||
silu = gate * tl.sigmoid(gate)
|
||||
out = silu * up
|
||||
tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _silu_mul_add_delta_kernel(
|
||||
x_ptr, # [T, 2N] gemm1 output, [gate || up]
|
||||
d_ptr, # [T, 2N] LoRA gate_up delta, same layout
|
||||
out_ptr, # [T, N]
|
||||
x_ptr,
|
||||
d_ptr,
|
||||
out_ptr,
|
||||
N,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
@@ -26,17 +49,7 @@ def _silu_mul_add_delta_kernel(
|
||||
block = tl.program_id(1)
|
||||
cols = block * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = cols < N
|
||||
|
||||
x_row = x_ptr + row * 2 * N
|
||||
d_row = d_ptr + row * 2 * N
|
||||
gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
gate += tl.load(d_row + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
up += tl.load(d_row + N + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
|
||||
silu = gate * tl.sigmoid(gate)
|
||||
out = silu * up
|
||||
tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask)
|
||||
_silu_mul_add_delta_row(x_ptr, d_ptr, out_ptr, N, row, cols, mask, mask)
|
||||
|
||||
|
||||
def silu_and_mul_add_delta(
|
||||
@@ -44,7 +57,7 @@ def silu_and_mul_add_delta(
|
||||
gate_up_delta: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""``out = silu(g + dg) * (u + du)`` over contiguous ``[T, 2N]`` inputs."""
|
||||
"""Both producers must define inactive route rows and disabled deltas as zero."""
|
||||
T, two_n = gemm1_out.shape
|
||||
N = two_n // 2
|
||||
assert gate_up_delta.shape == (T, two_n) and out.shape == (T, N)
|
||||
|
||||
@@ -24,6 +24,7 @@ def _direct_gate_expand_kernel(
|
||||
gate_b_ptr, # [S, E, 2I, R]
|
||||
topk_ids_ptr, # [M, topk]
|
||||
token_lora_mapping_ptr, # [M]
|
||||
adapter_enabled_ptr,
|
||||
output_ptr, # [M, topk, 2I]
|
||||
stride_as,
|
||||
stride_am,
|
||||
@@ -34,11 +35,13 @@ def _direct_gate_expand_kernel(
|
||||
stride_br,
|
||||
stride_oq,
|
||||
stride_on,
|
||||
stride_enabled,
|
||||
ROUTER_TOPK: tl.constexpr,
|
||||
INTERMEDIATE_SIZE: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
GATE_WIDTH: tl.constexpr,
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
HAS_ADAPTER_ENABLED: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
@@ -48,6 +51,15 @@ def _direct_gate_expand_kernel(
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
if HAS_ADAPTER_ENABLED:
|
||||
active = active & (
|
||||
tl.load(
|
||||
adapter_enabled_ptr + slot * stride_enabled,
|
||||
mask=(slot >= 0) & (slot < NUM_SLOTS),
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
@@ -91,6 +103,7 @@ def _direct_down_shrink_kernel(
|
||||
down_a_ptr, # [S, E, R, I]
|
||||
topk_ids_ptr, # [M, topk]
|
||||
token_lora_mapping_ptr, # [M]
|
||||
adapter_enabled_ptr,
|
||||
output_ptr, # [M, topk, R]
|
||||
stride_xq,
|
||||
stride_xk,
|
||||
@@ -100,10 +113,12 @@ def _direct_down_shrink_kernel(
|
||||
stride_ak,
|
||||
stride_oq,
|
||||
stride_or,
|
||||
stride_enabled,
|
||||
ROUTER_TOPK: tl.constexpr,
|
||||
INTERMEDIATE_SIZE: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
HAS_ADAPTER_ENABLED: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
@@ -112,6 +127,15 @@ def _direct_down_shrink_kernel(
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
if HAS_ADAPTER_ENABLED:
|
||||
active = active & (
|
||||
tl.load(
|
||||
adapter_enabled_ptr + slot * stride_enabled,
|
||||
mask=(slot >= 0) & (slot < NUM_SLOTS),
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_r = tl.arange(0, RANK)
|
||||
@@ -153,6 +177,8 @@ def direct_decode_gate_expand(
|
||||
topk_ids: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Expand shared gate/up rank vectors through directly selected experts.
|
||||
|
||||
@@ -163,6 +189,10 @@ def direct_decode_gate_expand(
|
||||
"""
|
||||
|
||||
num_tokens = topk_ids.shape[0]
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (gate_b.shape[0],)
|
||||
assert adapter_enabled.device == topk_ids.device
|
||||
assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64)
|
||||
if shared_intermediate.ndim == 2:
|
||||
shared_slot_stride = 0
|
||||
shared_token_stride = shared_intermediate.stride(0)
|
||||
@@ -180,6 +210,7 @@ def direct_decode_gate_expand(
|
||||
gate_b,
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
adapter_enabled,
|
||||
output_flat,
|
||||
shared_slot_stride,
|
||||
shared_token_stride,
|
||||
@@ -190,11 +221,13 @@ def direct_decode_gate_expand(
|
||||
gate_b.stride(3),
|
||||
output_flat.stride(0),
|
||||
output_flat.stride(1),
|
||||
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
|
||||
ROUTER_TOPK=ROUTER_TOPK,
|
||||
INTERMEDIATE_SIZE=intermediate_size,
|
||||
RANK=RANK,
|
||||
GATE_WIDTH=gate_width,
|
||||
NUM_SLOTS=gate_b.shape[0],
|
||||
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
|
||||
BLOCK_M=_BLOCK_M,
|
||||
BLOCK_N=_GATE_BLOCK_N,
|
||||
num_warps=4,
|
||||
@@ -208,6 +241,8 @@ def direct_decode_down_shrink(
|
||||
topk_ids: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
adapter_enabled: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Shrink directly routed activations through per-expert LoRA-A.
|
||||
|
||||
@@ -216,6 +251,10 @@ def direct_decode_down_shrink(
|
||||
"""
|
||||
|
||||
num_tokens = topk_ids.shape[0]
|
||||
if adapter_enabled is not None:
|
||||
assert adapter_enabled.shape == (down_a.shape[0],)
|
||||
assert adapter_enabled.device == topk_ids.device
|
||||
assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64)
|
||||
intermediate_size = activation.shape[1]
|
||||
output_flat = output.view(num_tokens * ROUTER_TOPK, RANK)
|
||||
num_routes = topk_ids.numel()
|
||||
@@ -224,6 +263,7 @@ def direct_decode_down_shrink(
|
||||
down_a,
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
adapter_enabled,
|
||||
output_flat,
|
||||
activation.stride(0),
|
||||
activation.stride(1),
|
||||
@@ -233,10 +273,12 @@ def direct_decode_down_shrink(
|
||||
down_a.stride(3),
|
||||
output_flat.stride(0),
|
||||
output_flat.stride(1),
|
||||
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
|
||||
ROUTER_TOPK=ROUTER_TOPK,
|
||||
INTERMEDIATE_SIZE=intermediate_size,
|
||||
RANK=RANK,
|
||||
NUM_SLOTS=down_a.shape[0],
|
||||
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
|
||||
BLOCK_M=_BLOCK_M,
|
||||
BLOCK_K=_DOWN_BLOCK_K,
|
||||
num_warps=4,
|
||||
|
||||
@@ -209,6 +209,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
lora_info,
|
||||
) -> StandardCombineInput:
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
_prewarm_per_expert_lora_routes,
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
@@ -233,7 +234,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
topk_weights = topk_output.topk_weights
|
||||
if (
|
||||
topk_ids.ndim != 2
|
||||
or topk_weights.shape != topk_ids.shape
|
||||
@@ -345,8 +346,25 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
!= lora_info.gate_up_lora_b_weights.shape[-1]
|
||||
)
|
||||
# Allocate and prewarm on main; side streams launch kernels only.
|
||||
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
|
||||
if not direct_decode:
|
||||
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
|
||||
paired_routing = (
|
||||
factored_shared_outer
|
||||
and not direct_decode
|
||||
and _prewarm_per_expert_lora_routes(
|
||||
hidden_states,
|
||||
lora_info.down_lora_a_weights,
|
||||
lora_info.gate_up_lora_b_weights,
|
||||
topk_ids,
|
||||
lora_info.token_lora_mapping,
|
||||
routing_cache,
|
||||
shrink_lora_b=lora_info.down_lora_b_weights,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
include_inactive_routes=True,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
)
|
||||
)
|
||||
if not direct_decode and not paired_routing:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=gate_up_delta,
|
||||
hidden_states=hidden_states,
|
||||
@@ -359,7 +377,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="routing",
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_a_routing=not factored_shared_outer,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
@@ -380,7 +400,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=True,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=collapsed_routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="routing",
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_a_routing=True,
|
||||
prewarm_b_routing=False,
|
||||
local_expert_offset=0,
|
||||
@@ -402,7 +424,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
)
|
||||
)
|
||||
# Prewarm down routing and allocate its intermediate on main.
|
||||
if not direct_decode:
|
||||
if not direct_decode and not paired_routing:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=gate_up_delta,
|
||||
hidden_states=hidden_states,
|
||||
@@ -415,8 +437,10 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="routing",
|
||||
prewarm_b_routing=not factored_shared_outer,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
)
|
||||
@@ -436,8 +460,10 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=True,
|
||||
routing_cache=collapsed_routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="routing",
|
||||
prewarm_a_routing=False,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_b_routing=True,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
@@ -476,7 +502,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=True,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=collapsed_routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="shrink",
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_b_routing=False,
|
||||
intermediate_buffer=gate_up_intermediate,
|
||||
local_expert_offset=0,
|
||||
@@ -495,6 +523,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
topk_ids,
|
||||
lora_info.token_lora_mapping,
|
||||
gate_up_delta,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
)
|
||||
return
|
||||
merged_experts_fused_moe_lora_add(
|
||||
@@ -509,7 +538,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=True,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
fuse_add_to_output=False,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
use_direct_expand_add=True,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
@@ -530,7 +561,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
|
||||
experts_shared_outer_loras_b=False,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
fuse_add_to_output=False,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
use_direct_expand_add=use_direct_expand,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
@@ -552,7 +585,11 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
# Start alignment after the side-stream fork. EP IDs are already localized;
|
||||
# the wrapper owns Marlin's extra sentinel bucket.
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, block_size_m, E
|
||||
topk_ids,
|
||||
block_size_m,
|
||||
E,
|
||||
return_inactive_prefix_count=not ep_active,
|
||||
map_invalid_experts_to_sentinel=False,
|
||||
)
|
||||
|
||||
# A cached workspace can outlive its CUDA graph-private pool.
|
||||
@@ -572,13 +609,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
)
|
||||
|
||||
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -597,7 +630,10 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
moe_block_size=block_size_m,
|
||||
top_k=topk,
|
||||
mul_topk_weights=False,
|
||||
zero_inactive_output=True,
|
||||
is_ep=ep_active,
|
||||
expert_ids_are_sorted=not ep_active,
|
||||
has_inactive_prefix_count=not ep_active,
|
||||
b_q_type=scalar_type1,
|
||||
size_m=M,
|
||||
size_n=2 * N,
|
||||
@@ -616,6 +652,8 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
if run_lora:
|
||||
if lora_event is not None:
|
||||
torch.cuda.current_stream().wait_event(lora_event)
|
||||
# Both private producers define dropped routes as zero; the runtime
|
||||
# activation therefore needs no routing or adapter metadata reads.
|
||||
silu_and_mul_add_delta(
|
||||
intermediate_cache1.view(-1, 2 * N),
|
||||
gate_up_delta.view(-1, 2 * N),
|
||||
@@ -641,6 +679,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
topk_ids,
|
||||
lora_info.token_lora_mapping,
|
||||
down_intermediate,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
)
|
||||
else:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
@@ -655,7 +694,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="shrink",
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_b_routing=not factored_shared_outer,
|
||||
intermediate_buffer=down_intermediate,
|
||||
local_expert_offset=0,
|
||||
@@ -673,6 +714,8 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
down_rank_sum,
|
||||
routed_scaling_factor,
|
||||
block_m=_weighted_rank_sum_block_m(M),
|
||||
topk_ids=topk_ids,
|
||||
inactive_ranks_are_zero=True,
|
||||
)
|
||||
down_shrink_done.record()
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
@@ -682,6 +725,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: down (marlin).
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
@@ -701,7 +747,10 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
moe_block_size=block_size_m,
|
||||
top_k=1,
|
||||
mul_topk_weights=True,
|
||||
zero_inactive_output=True,
|
||||
is_ep=ep_active,
|
||||
expert_ids_are_sorted=not ep_active,
|
||||
has_inactive_prefix_count=not ep_active,
|
||||
b_q_type=scalar_type2,
|
||||
size_m=M * topk,
|
||||
size_n=K,
|
||||
@@ -723,6 +772,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
topk_ids,
|
||||
lora_info.token_lora_mapping,
|
||||
down_intermediate,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
)
|
||||
else:
|
||||
down_intermediate.zero_()
|
||||
@@ -738,7 +788,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=True,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
stage="shrink",
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
prewarm_b_routing=False,
|
||||
intermediate_buffer=down_intermediate,
|
||||
local_expert_offset=0,
|
||||
@@ -760,7 +812,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
local_expert_offset=0,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
local_num_experts=E,
|
||||
zero_intermediate=lora_info.experts_shared_outer_loras,
|
||||
)
|
||||
@@ -782,6 +836,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
output,
|
||||
routed_scaling_factor,
|
||||
block_k=64,
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
else:
|
||||
tail_block_m, tail_block_k = fused_base_shared_lora_reduce_config(M)
|
||||
@@ -794,6 +849,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
routed_scaling_factor,
|
||||
block_m=tail_block_m,
|
||||
block_k=tail_block_k,
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
elif routed_scaling_factor == 1.0 and M <= 512:
|
||||
torch.sum(intermediate_cache3, dim=1, out=output)
|
||||
@@ -810,6 +866,8 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
down_rank_sum,
|
||||
routed_scaling_factor,
|
||||
block_m=_weighted_rank_sum_block_m(M),
|
||||
topk_ids=topk_ids,
|
||||
inactive_ranks_are_zero=True,
|
||||
)
|
||||
else:
|
||||
# Wait only at the shared-B consumer.
|
||||
@@ -830,7 +888,9 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=True,
|
||||
routing_cache=collapsed_routing_cache,
|
||||
include_inactive_routes=True,
|
||||
fuse_add_to_output=True,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
use_direct_expand_add=False,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
@@ -854,8 +914,10 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
experts_shared_outer_loras_a=False,
|
||||
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
|
||||
routing_cache=routing_cache,
|
||||
include_inactive_routes=True,
|
||||
fuse_add_to_output=False,
|
||||
fuse_sum_all_reduce=True,
|
||||
adapter_enabled=lora_info.adapter_enabled,
|
||||
use_direct_expand_add=lora_info.max_lora_rank <= 64,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
|
||||
@@ -16,6 +16,7 @@ import triton.language as tl
|
||||
def _weighted_topk_rank_sum_kernel(
|
||||
routed_rank_ptr, # [M, topk, R]
|
||||
topk_weights_ptr, # [M, topk]
|
||||
topk_ids_ptr,
|
||||
output_ptr, # [M, R]
|
||||
M,
|
||||
R,
|
||||
@@ -25,9 +26,13 @@ def _weighted_topk_rank_sum_kernel(
|
||||
stride_rr,
|
||||
stride_wm,
|
||||
stride_wk,
|
||||
stride_im,
|
||||
stride_ik,
|
||||
stride_om,
|
||||
stride_or,
|
||||
TOPK: tl.constexpr,
|
||||
HAS_TOPK_IDS: tl.constexpr,
|
||||
INACTIVE_RANKS_ARE_ZERO: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
):
|
||||
@@ -40,12 +45,23 @@ def _weighted_topk_rank_sum_kernel(
|
||||
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32)
|
||||
for k in tl.static_range(TOPK):
|
||||
route_mask = token_mask
|
||||
if HAS_TOPK_IDS:
|
||||
route = tl.load(
|
||||
topk_ids_ptr + token * stride_im + k * stride_ik,
|
||||
mask=token_mask,
|
||||
other=-1,
|
||||
)
|
||||
route_mask = route_mask & (route >= 0)
|
||||
rank_load_mask = mask & route_mask[:, None]
|
||||
if INACTIVE_RANKS_ARE_ZERO:
|
||||
rank_load_mask = mask
|
||||
routed_rank = tl.load(
|
||||
routed_rank_ptr
|
||||
+ token[:, None] * stride_rm
|
||||
+ k * stride_rk
|
||||
+ rank[None, :] * stride_rr,
|
||||
mask=mask,
|
||||
mask=rank_load_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight = tl.load(
|
||||
@@ -53,6 +69,8 @@ def _weighted_topk_rank_sum_kernel(
|
||||
mask=token_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
if HAS_TOPK_IDS:
|
||||
weight = tl.where(route_mask, weight, 0.0)
|
||||
accumulator += routed_rank * weight[:, None]
|
||||
|
||||
accumulator *= scale
|
||||
@@ -69,18 +87,33 @@ def weighted_topk_rank_sum(
|
||||
output: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
inactive_ranks_are_zero: bool = False,
|
||||
block_m: int,
|
||||
) -> None:
|
||||
"""Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``."""
|
||||
"""Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``.
|
||||
|
||||
``inactive_ranks_are_zero`` certifies that the rank producer writes literal
|
||||
zero for every dropped route. Original IDs select inactive scores to zero
|
||||
before multiplication, excluding NaNs from those scores. The default also
|
||||
masks rank loads and admits poisoned private payloads.
|
||||
"""
|
||||
|
||||
assert routed_rank.ndim == 3
|
||||
assert topk_weights.shape == routed_rank.shape[:2]
|
||||
assert output.shape == (routed_rank.shape[0], routed_rank.shape[2])
|
||||
assert routed_rank.is_contiguous() and topk_weights.is_contiguous()
|
||||
assert output.is_contiguous()
|
||||
if inactive_ranks_are_zero:
|
||||
assert topk_ids is not None
|
||||
if routed_rank.shape[0] == 0:
|
||||
return
|
||||
|
||||
if topk_ids is not None:
|
||||
assert topk_ids.shape == topk_weights.shape
|
||||
assert topk_ids.device == routed_rank.device
|
||||
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||
|
||||
rank = routed_rank.shape[2]
|
||||
assert block_m in (1, 2)
|
||||
block_rank = max(16, triton.next_power_of_2(rank))
|
||||
@@ -89,6 +122,7 @@ def weighted_topk_rank_sum(
|
||||
](
|
||||
routed_rank,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
output,
|
||||
routed_rank.shape[0],
|
||||
rank,
|
||||
@@ -98,9 +132,13 @@ def weighted_topk_rank_sum(
|
||||
routed_rank.stride(2),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
topk_ids.stride(0) if topk_ids is not None else 0,
|
||||
topk_ids.stride(1) if topk_ids is not None else 0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
TOPK=routed_rank.shape[1],
|
||||
HAS_TOPK_IDS=topk_ids is not None,
|
||||
INACTIVE_RANKS_ARE_ZERO=inactive_ranks_are_zero,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_R=block_rank,
|
||||
num_warps=1,
|
||||
@@ -112,6 +150,7 @@ def _fused_base_shared_lora_reduce_kernel(
|
||||
routed_base_ptr, # [M, topk, K], already weighted by the base MoE
|
||||
routed_rank_ptr, # [M, topk, R]
|
||||
topk_weights_ptr, # [M, topk]
|
||||
topk_ids_ptr,
|
||||
shared_b_ptr, # [K, R]
|
||||
output_ptr, # [M, K]
|
||||
M,
|
||||
@@ -126,11 +165,14 @@ def _fused_base_shared_lora_reduce_kernel(
|
||||
stride_rr,
|
||||
stride_wm,
|
||||
stride_wk,
|
||||
stride_im,
|
||||
stride_ik,
|
||||
stride_sk,
|
||||
stride_sr,
|
||||
stride_om,
|
||||
stride_ok,
|
||||
TOPK: tl.constexpr,
|
||||
HAS_TOPK_IDS: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
@@ -147,12 +189,20 @@ def _fused_base_shared_lora_reduce_kernel(
|
||||
base_acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
|
||||
rank_acc = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32)
|
||||
for topk_idx in tl.static_range(TOPK):
|
||||
route_mask = token_mask
|
||||
if HAS_TOPK_IDS:
|
||||
route = tl.load(
|
||||
topk_ids_ptr + token * stride_im + topk_idx * stride_ik,
|
||||
mask=token_mask,
|
||||
other=-1,
|
||||
)
|
||||
route_mask = route_mask & (route >= 0)
|
||||
base = tl.load(
|
||||
routed_base_ptr
|
||||
+ token[:, None] * stride_bm
|
||||
+ topk_idx * stride_bk
|
||||
+ out_col[None, :] * stride_bn,
|
||||
mask=token_mask[:, None] & out_mask[None, :],
|
||||
mask=route_mask[:, None] & out_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
routed_rank = tl.load(
|
||||
@@ -160,12 +210,12 @@ def _fused_base_shared_lora_reduce_kernel(
|
||||
+ token[:, None] * stride_rm
|
||||
+ topk_idx * stride_rk
|
||||
+ rank[None, :] * stride_rr,
|
||||
mask=token_mask[:, None] & rank_mask[None, :],
|
||||
mask=route_mask[:, None] & rank_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight = tl.load(
|
||||
topk_weights_ptr + token * stride_wm + topk_idx * stride_wk,
|
||||
mask=token_mask,
|
||||
mask=route_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
base_acc += base
|
||||
@@ -200,6 +250,7 @@ def fused_base_shared_lora_reduce(
|
||||
output: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
block_m: int,
|
||||
block_k: int,
|
||||
) -> None:
|
||||
@@ -237,6 +288,11 @@ def fused_base_shared_lora_reduce(
|
||||
if routed_base.shape[0] == 0:
|
||||
return
|
||||
|
||||
if topk_ids is not None:
|
||||
assert topk_ids.shape == topk_weights.shape
|
||||
assert topk_ids.device == routed_rank.device
|
||||
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||
|
||||
rank = routed_rank.shape[2]
|
||||
assert block_m in (1, 2, 4, 8)
|
||||
assert block_k in (32, 64, 128)
|
||||
@@ -250,6 +306,7 @@ def fused_base_shared_lora_reduce(
|
||||
routed_base,
|
||||
routed_rank,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
shared_b,
|
||||
output,
|
||||
routed_base.shape[0],
|
||||
@@ -264,11 +321,14 @@ def fused_base_shared_lora_reduce(
|
||||
routed_rank.stride(2),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
topk_ids.stride(0) if topk_ids is not None else 0,
|
||||
topk_ids.stride(1) if topk_ids is not None else 0,
|
||||
shared_b.stride(0),
|
||||
shared_b.stride(1),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
TOPK=routed_base.shape[1],
|
||||
HAS_TOPK_IDS=topk_ids is not None,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
BLOCK_R=block_rank,
|
||||
@@ -284,6 +344,7 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
|
||||
routed_base_ptr,
|
||||
routed_rank_ptr,
|
||||
topk_weights_ptr,
|
||||
topk_ids_ptr,
|
||||
shared_b_ptr, # [S, K, R]
|
||||
token_lora_mapping_ptr,
|
||||
output_ptr,
|
||||
@@ -298,12 +359,15 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
|
||||
stride_rr,
|
||||
stride_wm,
|
||||
stride_wk,
|
||||
stride_im,
|
||||
stride_ik,
|
||||
stride_ss,
|
||||
stride_sk,
|
||||
stride_sr,
|
||||
stride_om,
|
||||
stride_ok,
|
||||
TOPK: tl.constexpr,
|
||||
HAS_TOPK_IDS: tl.constexpr,
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
@@ -317,12 +381,16 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
|
||||
base_acc = tl.zeros((1, BLOCK_K), dtype=tl.float32)
|
||||
rank_acc = tl.zeros((1, BLOCK_R), dtype=tl.float32)
|
||||
for topk_idx in tl.static_range(TOPK):
|
||||
route_active = tl.full((), True, tl.int1)
|
||||
if HAS_TOPK_IDS:
|
||||
route = tl.load(topk_ids_ptr + token * stride_im + topk_idx * stride_ik)
|
||||
route_active = route >= 0
|
||||
base = tl.load(
|
||||
routed_base_ptr
|
||||
+ token * stride_bm
|
||||
+ topk_idx * stride_bk
|
||||
+ out_col[None, :] * stride_bn,
|
||||
mask=out_mask[None, :],
|
||||
mask=route_active & out_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
routed_rank = tl.load(
|
||||
@@ -330,10 +398,14 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
|
||||
+ token * stride_rm
|
||||
+ topk_idx * stride_rk
|
||||
+ rank[None, :] * stride_rr,
|
||||
mask=rank_mask[None, :],
|
||||
mask=route_active & rank_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight = tl.load(topk_weights_ptr + token * stride_wm + topk_idx * stride_wk)
|
||||
weight = tl.load(
|
||||
topk_weights_ptr + token * stride_wm + topk_idx * stride_wk,
|
||||
mask=route_active,
|
||||
other=0.0,
|
||||
)
|
||||
base_acc += base
|
||||
rank_acc += routed_rank * weight
|
||||
|
||||
@@ -366,6 +438,7 @@ def fused_base_mapped_shared_lora_reduce(
|
||||
output: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
block_k: int,
|
||||
) -> None:
|
||||
"""Fused decode tail with a per-token shared-B adapter slot."""
|
||||
@@ -417,6 +490,11 @@ def fused_base_mapped_shared_lora_reduce(
|
||||
if routed_base.shape[0] == 0:
|
||||
return
|
||||
|
||||
if topk_ids is not None:
|
||||
assert topk_ids.shape == topk_weights.shape
|
||||
assert topk_ids.device == routed_rank.device
|
||||
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||
|
||||
assert block_k in (32, 64, 128)
|
||||
block_rank = max(16, triton.next_power_of_2(routed_rank.shape[2]))
|
||||
_fused_base_mapped_shared_lora_reduce_kernel[
|
||||
@@ -425,6 +503,7 @@ def fused_base_mapped_shared_lora_reduce(
|
||||
routed_base,
|
||||
routed_rank,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
shared_b_view,
|
||||
token_lora_mapping,
|
||||
output,
|
||||
@@ -439,12 +518,15 @@ def fused_base_mapped_shared_lora_reduce(
|
||||
routed_rank.stride(2),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
topk_ids.stride(0) if topk_ids is not None else 0,
|
||||
topk_ids.stride(1) if topk_ids is not None else 0,
|
||||
shared_b_view.stride(0),
|
||||
shared_b_view.stride(1),
|
||||
shared_b_view.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
TOPK=routed_base.shape[1],
|
||||
HAS_TOPK_IDS=topk_ids is not None,
|
||||
NUM_SLOTS=shared_b.shape[0],
|
||||
BLOCK_K=block_k,
|
||||
BLOCK_R=block_rank,
|
||||
|
||||
@@ -65,6 +65,9 @@ def _moe_lora_expand_add_kernel(
|
||||
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
|
||||
if pid >= num_pid_m * num_pid_n:
|
||||
return
|
||||
|
||||
num_pid_in_group = GROUP_SIZE_M * num_pid_n
|
||||
group_id = pid // num_pid_in_group
|
||||
first_pid_m = group_id * GROUP_SIZE_M
|
||||
@@ -72,9 +75,6 @@ def _moe_lora_expand_add_kernel(
|
||||
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
|
||||
pid_n = (pid % num_pid_in_group) // group_size_m
|
||||
|
||||
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
|
||||
return
|
||||
|
||||
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
|
||||
token_mask = offs_token < num_valid_tokens
|
||||
|
||||
Reference in New Issue
Block a user