[Fix] lora: Preserve MoE projection identity and routing lifetimes (#45681)

This commit is contained in:
Cheng Wan
2026-10-23 12:32:55 -07:00
committed by GitHub
parent ac3db731b3
commit d0022e1df3
18 changed files with 2561 additions and 483 deletions
@@ -48,11 +48,21 @@ using Vec = int4;
namespace moe_lora_merged {
template <bool HasAdapterEnabled>
__device__ __forceinline__ bool
adapter_is_enabled(int lora_id, const int32_t* adapter_enabled, int num_slots, int64_t enabled_stride) {
if (lora_id < 0 || lora_id >= num_slots) return false;
if constexpr (HasAdapterEnabled) {
return adapter_enabled[lora_id * enabled_stride] != 0;
}
return true;
}
// Inline mirror of _fused_virtual_topk_ids_kernel (virtual_experts.py). Returns
// the merged virtual expert id for flat slot `i` (range [-1, virtual_num_experts);
// -1 is the dropped/masked sentinel). The caller adds +1 to get the histogram
// bucket (sentinel -> bucket 0), matching the native +1 offset convention.
template <typename scalar_t>
template <typename scalar_t, bool HasAdapterEnabled>
__device__ __forceinline__ int compute_virtual_id(
const scalar_t* __restrict__ topk_ids,
const int32_t* __restrict__ token_lora_mapping,
@@ -63,27 +73,32 @@ __device__ __forceinline__ int compute_virtual_id(
int local_num_experts,
bool ep_local,
bool shared_outer,
bool compact) {
bool compact,
const int32_t* __restrict__ adapter_enabled,
int num_slots,
int64_t enabled_stride) {
int m = static_cast<int>(i) / top_k;
int lora_id = token_lora_mapping[m];
bool mask_val = lora_id >= 0;
int safe_lora = lora_id > 0 ? lora_id : 0;
bool mask_val = adapter_is_enabled<HasAdapterEnabled>(lora_id, adapter_enabled, num_slots, enabled_stride);
int base = shared_outer ? 0 : static_cast<int>(topk_ids[i]);
scalar_t original_id = topk_ids[i];
if (!mask_val || original_id < 0) return -1;
if (!shared_outer && original_id >= num_experts_for_weight) return -1;
int base = shared_outer ? 0 : static_cast<int>(original_id);
if (ep_local) {
bool owned = base >= local_expert_offset && base < local_expert_offset + local_num_experts;
base = owned ? base : -1;
}
if (!mask_val || base < 0) return -1;
if (base < 0) return -1;
// compact: dense LOCAL expert id in [0, local_num_experts) so the histogram
// spans only local_num_experts buckets instead of the full global virtual
// space (337/385 empty under EP). Assumes max_loras==1 (safe_lora shift is 0;
// space (337/385 empty under EP). Assumes max_loras==1 (adapter shift is 0;
// the wrapper guards). expert_ids is converted back to global at write time.
if (compact) return base - local_expert_offset;
return base + safe_lora * num_experts_for_weight;
return base + lora_id * num_experts_for_weight;
}
template <typename scalar_t>
template <typename scalar_t, bool HasAdapterEnabled>
__global__ void count_and_sort_expert_tokens_kernel(
const scalar_t* __restrict__ topk_ids,
const int32_t* __restrict__ token_lora_mapping,
@@ -97,12 +112,15 @@ __global__ void count_and_sort_expert_tokens_kernel(
bool ep_local,
bool shared_outer,
bool do_skip,
bool compact) {
bool compact,
const int32_t* __restrict__ adapter_enabled,
int num_slots,
int64_t enabled_stride) {
const size_t tid = blockIdx.x * blockDim.x + threadIdx.x;
const size_t stride = blockDim.x * gridDim.x;
for (size_t i = tid; i < numel; i += stride) {
int vid = compute_virtual_id<scalar_t>(
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
topk_ids,
token_lora_mapping,
i,
@@ -112,7 +130,10 @@ __global__ void count_and_sort_expert_tokens_kernel(
local_num_experts,
ep_local,
shared_outer,
compact);
compact,
adapter_enabled,
num_slots,
enabled_stride);
// EP skip: dropped/masked slots (vid < 0) produce no delta on this rank, so
// they never need a slot in sorted_token_ids -> skip the global atomicAdd
// (kills the sentinel-bucket-0 contention). When do_skip is off they fall
@@ -124,7 +145,7 @@ __global__ void count_and_sort_expert_tokens_kernel(
}
}
template <typename scalar_t>
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
__global__ void moe_align_block_size_kernel(
const scalar_t* __restrict__ topk_ids,
const int32_t* __restrict__ token_lora_mapping,
@@ -146,7 +167,10 @@ __global__ void moe_align_block_size_kernel(
bool ep_local,
bool shared_outer,
bool do_skip,
bool compact) {
bool compact,
const int32_t* __restrict__ adapter_enabled,
int num_slots,
int64_t enabled_stride) {
// Use a separate thread block to populate sorted_token_ids
if (blockIdx.x == 1) {
if (pad_sorted_token_ids) {
@@ -181,7 +205,7 @@ __global__ void moe_align_block_size_kernel(
__syncthreads();
for (size_t i = tid; i < numel; i += stride) {
int vid = compute_virtual_id<scalar_t>(
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
topk_ids,
token_lora_mapping,
i,
@@ -191,16 +215,20 @@ __global__ void moe_align_block_size_kernel(
local_num_experts,
ep_local,
shared_outer,
compact);
compact,
adapter_enabled,
num_slots,
enabled_stride);
// EP skip: dropped/masked slots don't increment any bucket (sentinel bucket
// 0 stays empty), so they never get a block and never reach count_and_sort.
if (!do_skip || vid >= 0) {
atomicAdd(&shared_counts[vid + 1], 1);
}
// token_lora_mask[m] = token_lora_mapping[m] >= 0, written once per row.
// Publish the same bounded adapter predicate used by both routing passes.
if (static_cast<int>(i) % top_k == 0) {
int m = static_cast<int>(i) / top_k;
token_lora_mask[m] = token_lora_mapping[m] >= 0;
token_lora_mask[m] =
adapter_is_enabled<HasAdapterEnabled>(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride);
}
}
@@ -235,6 +263,9 @@ __global__ void moe_align_block_size_kernel(
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
s_total_tokens_post_pad = prefix[num_experts];
*total_tokens_post_pad = s_total_tokens_post_pad;
if constexpr (WriteInactivePrefixCount) {
total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size);
}
}
__syncthreads();
@@ -277,9 +308,9 @@ __global__ void moe_align_block_size_kernel(
right = mid;
}
}
// compact buckets hold LOCAL expert ids; restore the global id (+offset) so
// the downstream GEMM still indexes the global contiguous LoRA weight.
expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0);
// Restore global IDs only for real compact buckets. The negative bucket
// stays -1 so overwrite consumers can zero it before touching weights.
expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0);
}
}
@@ -287,9 +318,9 @@ __global__ void moe_align_block_size_kernel(
// in ONE threadblock (one launch), eliminating the separate count_and_sort kernel
// AND its redundant re-computation of the virtual id (cached in shared `svids`).
// Only valid for small numel (the scatter is single-block); the wrapper routes
// large numel (prefill) to the 2-kernel path. do_skip is implied (this is the
// decode hot path); dropped slots are simply never scattered.
template <typename scalar_t>
// large numel (prefill) to the 2-kernel path. do_skip omits dropped slots;
// overwrite consumers can retain them for the existing negative-expert store.
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
__global__ void fused_align_scatter_kernel(
const scalar_t* __restrict__ topk_ids,
const int32_t* __restrict__ token_lora_mapping,
@@ -310,7 +341,10 @@ __global__ void fused_align_scatter_kernel(
bool ep_local,
bool shared_outer,
bool do_skip,
bool compact) {
bool compact,
const int32_t* __restrict__ adapter_enabled,
int num_slots,
int64_t enabled_stride) {
extern __shared__ int32_t smem[];
int32_t* shared_counts = smem; // [num_experts]
int32_t* prefix = shared_counts + num_experts; // [num_experts + 1]
@@ -345,7 +379,7 @@ __global__ void fused_align_scatter_kernel(
// Phase 2: histogram + cache the virtual id per slot + token_lora_mask.
for (size_t i = tid; i < numel; i += stride) {
int vid = compute_virtual_id<scalar_t>(
int vid = compute_virtual_id<scalar_t, HasAdapterEnabled>(
topk_ids,
token_lora_mapping,
i,
@@ -355,14 +389,18 @@ __global__ void fused_align_scatter_kernel(
local_num_experts,
ep_local,
shared_outer,
compact);
compact,
adapter_enabled,
num_slots,
enabled_stride);
svids[i] = vid;
if (!do_skip || vid >= 0) {
atomicAdd(&shared_counts[vid + 1], 1);
}
if (static_cast<int>(i) % top_k == 0) {
int m = static_cast<int>(i) / top_k;
token_lora_mask[m] = token_lora_mapping[m] >= 0;
token_lora_mask[m] =
adapter_is_enabled<HasAdapterEnabled>(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride);
}
}
__syncthreads();
@@ -387,6 +425,9 @@ __global__ void fused_align_scatter_kernel(
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
s_total_tokens_post_pad = prefix[num_experts];
*total_tokens_post_pad = s_total_tokens_post_pad;
if constexpr (WriteInactivePrefixCount) {
total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size);
}
}
__syncthreads();
if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0;
@@ -420,7 +461,7 @@ __global__ void fused_align_scatter_kernel(
right = mid;
}
}
expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0);
expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0);
}
if (tid < num_experts) cursor[tid] = prefix[tid];
__syncthreads();
@@ -437,7 +478,7 @@ __global__ void fused_align_scatter_kernel(
} // namespace moe_lora_merged
template <typename scalar_t>
template <typename scalar_t, bool HasAdapterEnabled, bool WriteInactivePrefixCount>
struct MoeLoraMergedAlignKernel {
static void
run(tvm::ffi::TensorView topk_ids,
@@ -458,7 +499,10 @@ struct MoeLoraMergedAlignKernel {
bool shared_outer,
bool do_skip,
bool compact,
bool fuse_scatter) {
bool fuse_scatter,
tvm::ffi::Optional<tvm::ffi::TensorView> adapter_enabled,
int64_t num_slots,
int64_t enabled_stride) {
using namespace host;
auto device = topk_ids.device();
@@ -475,12 +519,6 @@ struct MoeLoraMergedAlignKernel {
// shared memory); the v2 (>1024) regime keeps the old path via the wrapper.
RuntimeCheck(
num_experts <= 1024, "moe_lora_merged_align: num_experts (bucket count) must be <= 1024, got ", num_experts);
// compact buckets hold LOCAL ids and restore global expert ids as
// (left-2+offset). For the sentinel bucket 0 that yields (offset-1), NOT the
// -1 the GEMM expects to skip -- only safe when do_skip empties bucket 0.
RuntimeCheck(
!compact || do_skip, "moe_lora_merged_align: compact requires do_skip (sentinel bucket must be empty)");
const scalar_t* topk_ids_ptr = static_cast<const scalar_t*>(topk_ids.data_ptr());
const int32_t* tlm_ptr = static_cast<const int32_t*>(token_lora_mapping.data_ptr());
bool* token_lora_mask_ptr = static_cast<bool*>(token_lora_mask.data_ptr());
@@ -489,6 +527,21 @@ struct MoeLoraMergedAlignKernel {
int32_t* num_tokens_post_pad_ptr = static_cast<int32_t*>(num_tokens_post_pad.data_ptr());
int32_t* cumsum_buffer_ptr = static_cast<int32_t*>(cumsum_buffer.data_ptr());
size_t numel = topk_ids.numel();
const int32_t* enabled_ptr = nullptr;
if constexpr (HasAdapterEnabled) {
CHECK_HOST(adapter_enabled.has_value());
TensorMatcher({num_slots})
.with_dtype<int32_t>()
.with_device(device)
.with_strides({enabled_stride})
.verify(adapter_enabled.value());
enabled_ptr = static_cast<const int32_t*>(adapter_enabled.value().data_ptr());
}
TensorMatcher({WriteInactivePrefixCount ? 2 : 1})
.with_dtype<int32_t>()
.with_device(device)
.ensure_alignment(WriteInactivePrefixCount ? 8 : 4)
.verify(num_tokens_post_pad);
const size_t scan_size = host::round_up_pow2(static_cast<uint32_t>(num_experts));
@@ -497,7 +550,7 @@ struct MoeLoraMergedAlignKernel {
// shared for the scatter cursor [num_experts] and cached virtual ids [numel].
const size_t shmem =
(num_experts + (num_experts + 1) + scan_size + WARP_SIZE + num_experts + numel) * sizeof(int32_t);
auto fused = moe_lora_merged::fused_align_scatter_kernel<scalar_t>;
auto fused = moe_lora_merged::fused_align_scatter_kernel<scalar_t, HasAdapterEnabled, WriteInactivePrefixCount>;
LaunchKernel(dim3(1), dim3(threads), stream, shmem)(
fused,
topk_ids_ptr,
@@ -519,13 +572,17 @@ struct MoeLoraMergedAlignKernel {
ep_local,
shared_outer,
do_skip,
compact);
compact,
enabled_ptr,
static_cast<int>(num_slots),
enabled_stride);
return;
}
const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t);
auto align_kernel = moe_lora_merged::moe_align_block_size_kernel<scalar_t>;
auto align_kernel =
moe_lora_merged::moe_align_block_size_kernel<scalar_t, HasAdapterEnabled, WriteInactivePrefixCount>;
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
align_kernel,
topk_ids_ptr,
@@ -548,14 +605,19 @@ struct MoeLoraMergedAlignKernel {
ep_local,
shared_outer,
do_skip,
compact);
compact,
enabled_ptr,
static_cast<int>(num_slots),
enabled_stride);
if (numel == 0) return;
const int block_threads = std::min(256, threads);
const int num_blocks = (numel + block_threads - 1) / block_threads;
const int max_blocks = 65535;
const int actual_blocks = std::min(num_blocks, max_blocks);
auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel<scalar_t>;
auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel<scalar_t, HasAdapterEnabled>;
LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)(
sort_kernel,
topk_ids_ptr,
@@ -570,7 +632,10 @@ struct MoeLoraMergedAlignKernel {
ep_local,
shared_outer,
do_skip,
compact);
compact,
enabled_ptr,
static_cast<int>(num_slots),
enabled_stride);
}
};
@@ -50,18 +50,17 @@ def route_bucket_ids(
# Invalid slots/groups map to -1. With one group per slot, any nonnegative
# group folds into that slot; otherwise bound it to the bucket table.
token_ids = pair_ids // WIDTH
slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1).to(tl.int32)
slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1)
live = (slots >= 0) & (slots < MAX_LORAS)
slots = slots.to(tl.int32)
if HAS_GROUPS:
groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1).to(
tl.int32
)
groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1)
live = live & (groups >= 0)
if GROUPS_PER_SLOT == 1:
buckets = slots
else:
live = live & (groups < GROUPS_PER_SLOT)
buckets = slots * GROUPS_PER_SLOT + groups
buckets = slots * GROUPS_PER_SLOT + groups.to(tl.int32)
else:
buckets = slots
return tl.where(live, buckets, -1)
@@ -11,8 +11,16 @@ if TYPE_CHECKING:
@cache_once
def _jit_module(dtype: torch.dtype) -> Module:
args = make_cpp_args(dtype)
def _jit_module(
dtype: torch.dtype,
has_adapter_enabled: bool = False,
write_inactive_prefix_count: bool = False,
) -> Module:
args = make_cpp_args(
dtype,
has_adapter_enabled,
write_inactive_prefix_count,
)
return load_jit(
"moe_lora_merged_align",
*args,
@@ -35,6 +43,8 @@ def moe_lora_merged_align(
do_skip: bool = True,
compact: bool = False,
fuse_scatter: Optional[bool] = None,
adapter_enabled: torch.Tensor | None = None,
return_inactive_prefix_count: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""Fused replacement for (_fused_virtual_topk_ids + _align_block_size) on the
merged-virtual-expert LoRA path.
@@ -44,13 +54,17 @@ def moe_lora_merged_align(
Returns (sorted_token_ids, expert_ids, num_tokens_post_padded,
token_lora_mask, virtual_num_experts).
With ``return_inactive_prefix_count``, the count is an aligned int32 pair
[total padded tokens, leading negative-expert blocks]; do_skip publishes zero.
"""
device = topk_ids.device
flat_topk_ids = topk_ids.reshape(-1)
if flat_topk_ids.dtype == torch.int64:
flat_topk_ids = flat_topk_ids.to(torch.int32)
M, top_k = topk_ids.shape
numel = M * top_k
if adapter_enabled is not None:
assert adapter_enabled.shape == (max_loras,)
assert adapter_enabled.dtype == torch.int32
assert adapter_enabled.device == device
num_experts_for_weight = 1 if shared_outer else num_experts
virtual_num_experts = num_experts_for_weight * max_loras
@@ -97,13 +111,19 @@ def moe_lora_merged_align(
sorted_alloc = (max_num_tokens_padded + 3) & ~3
sorted_token_ids = torch.empty((sorted_alloc,), dtype=torch.int32, device=device)
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
num_tokens_post_pad = torch.empty(
(2 if return_inactive_prefix_count else 1,), dtype=torch.int32, device=device
)
# No memset: the align kernel writes cumsum[0..num_buckets] before
# count_and_sort reads it (same as moe_align_block_size.py's empty cumsum).
cumsum_buffer = torch.empty((bucket_experts + 2,), dtype=torch.int32, device=device)
token_lora_mask = torch.empty((M,), dtype=torch.bool, device=device)
module = _jit_module(flat_topk_ids.dtype)
module = _jit_module(
flat_topk_ids.dtype,
adapter_enabled is not None,
return_inactive_prefix_count,
)
module.moe_lora_merged_align(
flat_topk_ids,
token_lora_mapping,
@@ -124,6 +144,9 @@ def moe_lora_merged_align(
do_skip,
compact_eff,
fuse_eff,
adapter_enabled,
max_loras,
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
)
return (
@@ -22,26 +22,25 @@ def _fused_virtual_topk_ids_kernel(
token_lora_mapping_ptr,
virtual_topk_ids_ptr,
token_lora_mask_ptr,
adapter_enabled_ptr,
stride_enabled,
num_experts_for_weight: tl.constexpr,
SHARED_OUTER: tl.constexpr,
M,
top_k: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
local_expert_offset: tl.constexpr,
local_num_experts: tl.constexpr,
EP_LOCAL: tl.constexpr,
NUM_SLOTS: tl.constexpr,
HAS_ADAPTER_ENABLED: tl.constexpr,
ENABLE_PDL: tl.constexpr = False,
):
"""
Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel.
Map active slots and valid base experts to virtual weight indices.
For each (m, k):
lora_id = token_lora_mapping[m]
mask[m] = (lora_id >= 0)
safe_lora = max(lora_id, 0)
if shared_outer: (handled by num_experts_for_weight == 0 sentinel)
virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora)
else:
virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight
Invalid slots or base experts map to -1. Shared weights accept any
nonnegative base expert and use the adapter slot as the weight index.
"""
pid = tl.program_id(0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
@@ -57,8 +56,20 @@ def _fused_virtual_topk_ids_kernel(
# k = offs % top_k # not needed directly
lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0)
mask_val = lora_id >= 0
safe_lora = tl.maximum(lora_id, 0)
mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS)
if HAS_ADAPTER_ENABLED:
if NUM_SLOTS == 1:
mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0)
else:
mask_val = mask_val & (
tl.load(
adapter_enabled_ptr + lora_id * stride_enabled,
mask=valid & mask_val,
other=0,
)
!= 0
)
safe_lora = tl.where(mask_val, lora_id, 0)
base = tl.load(topk_ids_ptr + offs, mask=valid, other=0)
if EP_LOCAL:
@@ -70,13 +81,13 @@ def _fused_virtual_topk_ids_kernel(
base < local_expert_offset + local_num_experts
)
base = tl.where(owned, base, -1)
# Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after
# EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on
# a real virtual-expert slot belonging to another adapter and trigger OOB
# loads in downstream LoRA kernels.
shifted = base + safe_lora * num_experts_for_weight
result = tl.where(base < 0, base, shifted)
result = tl.where(mask_val, result, -1)
valid_base = base >= 0
if not SHARED_OUTER:
valid_base = valid_base & (base < num_experts_for_weight)
expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0)
shifted = expert + safe_lora * num_experts_for_weight
# All align providers use one sentinel bucket; borrowed route IDs stay intact.
result = tl.where(mask_val & valid_base, shifted, -1)
tl.store(virtual_topk_ids_ptr + offs, result, mask=valid)
# Write mask once per row (at first k position)
@@ -97,48 +108,52 @@ def _fused_virtual_topk_ids(
max_loras: int,
local_expert_offset: int = 0,
local_num_experts: int | None = None,
adapter_enabled: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, int]:
"""
Returns virtual topk_ids, token_lora_mask, and virtual_num_experts.
`local_num_experts` (< `num_experts`) enables the EP mask on the per-expert
path: non-owned `topk_ids` slots are dropped to the -1 sentinel in-kernel.
The shared-outer path routes by lora id, not expert, so it is never masked
(else non-owned-but-valid tokens would be dropped on ranks > 0).
The shared-outer path drops negative routes but does not apply the local
expert range (its weights are shared by every owned expert).
"""
M, top_k = topk_ids.shape
device = topk_ids.device
if shared_outer:
num_experts_for_weight = 1
# For shared_outer, we need topk_ids to be zeros
zero_topk = torch.zeros_like(topk_ids)
input_topk = zero_topk
ep_local = False
else:
num_experts_for_weight = num_experts
input_topk = topk_ids
ep_local = local_num_experts is not None and local_num_experts < num_experts
num_experts_for_weight = 1 if shared_outer else num_experts
ep_local = (
not shared_outer
and local_num_experts is not None
and local_num_experts < num_experts
)
virtual_topk_ids = torch.empty_like(topk_ids)
token_lora_mask = torch.empty(M, dtype=torch.bool, device=device)
if M * top_k == 0:
return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras
BLOCK_SIZE = 1024
grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,)
enable_pdl, pdl_kwargs = get_pdl_launch_metadata()
_fused_virtual_topk_ids_kernel[grid](
input_topk,
topk_ids,
token_lora_mapping,
virtual_topk_ids,
token_lora_mask,
adapter_enabled,
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
num_experts_for_weight,
shared_outer,
M,
top_k,
BLOCK_SIZE,
local_expert_offset,
local_num_experts if local_num_experts is not None else 0,
ep_local,
NUM_SLOTS=max_loras,
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
ENABLE_PDL=enable_pdl,
**pdl_kwargs,
)
@@ -147,61 +162,6 @@ def _fused_virtual_topk_ids(
return virtual_topk_ids, token_lora_mask, virtual_num_experts
@triton.jit
def _fused_sanitize_expert_ids_kernel(
expert_ids_ptr,
output_ptr,
num_virtual_experts,
N,
BLOCK_SIZE: tl.constexpr,
ENABLE_PDL: tl.constexpr = False,
):
pid = tl.program_id(0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
valid = offs < N
# GDC wait: expert_ids comes from the immediately preceding align kernel;
# wait before consuming it.
if ENABLE_PDL:
tl.extra.cuda.gdc_wait()
eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0)
result = tl.where(eid < num_virtual_experts, eid, -1)
tl.store(output_ptr + offs, result, mask=valid)
# All work is done; hint the runtime to launch the dependent kernel.
if ENABLE_PDL:
tl.extra.cuda.gdc_launch_dependents()
def fused_sanitize_expert_ids(
expert_ids: torch.Tensor,
num_virtual_experts: int,
) -> torch.Tensor:
"""
Sanitize expert_ids by replacing values >= num_virtual_experts with -1.
Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1.
"""
N = expert_ids.numel()
output = torch.empty_like(expert_ids)
BLOCK_SIZE = 1024
grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,)
enable_pdl, pdl_kwargs = get_pdl_launch_metadata()
_fused_sanitize_expert_ids_kernel[grid](
expert_ids,
output,
num_virtual_experts,
N,
BLOCK_SIZE,
ENABLE_PDL=enable_pdl,
**pdl_kwargs,
)
return output
@triton.jit
def _moe_lora_shrink_splitk_kernel(
# Pointers
@@ -238,8 +198,8 @@ def _moe_lora_shrink_splitk_kernel(
pid_mn = pid // SPLIT_K
# GDC wait: the routing buffers (sorted_token_ids/expert_ids/
# num_tokens_post_padded) come from the immediately preceding align and
# sanitize kernels; wait before consuming them.
# num_tokens_post_padded) come from routing preparation; wait before
# consuming them.
if ENABLE_PDL:
tl.extra.cuda.gdc_wait()
@@ -247,6 +207,9 @@ def _moe_lora_shrink_splitk_kernel(
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
if pid_mn >= num_pid_m * num_pid_n:
return
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid_mn // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
@@ -254,9 +217,6 @@ def _moe_lora_shrink_splitk_kernel(
pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m)
pid_n = (pid_mn % num_pid_in_group) // group_size_m
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
return
# Token routing (same pattern as fused_moe_triton_kernels)
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
@@ -417,6 +377,180 @@ def _merged_experts_fused_moe_lora_add_fake(
return
def _get_stage_config(
weight_shape: tuple[int, ...],
stage_top_k: int,
dtype: torch.dtype,
num_tokens: int,
) -> dict[str, Any]:
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import (
get_config_dtype_str,
try_get_optimal_moe_config,
)
config_dtype = get_config_dtype_str(dtype=dtype)
get_config_func = functools.partial(
try_get_optimal_moe_config,
weight_shape,
weight_shape,
stage_top_k,
config_dtype,
)
try:
cfg = get_config_func(num_tokens)
except ValueError:
K_dim = weight_shape[2]
N_dim = weight_shape[1]
if K_dim >= 1024:
default_block_k = 256
elif K_dim >= 64:
default_block_k = 64
else:
default_block_k = max(16, K_dim)
cfg = {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": min(64, max(16, N_dim)),
"BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)),
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4,
}
return cfg
def _get_shrink_stage_config(
weight_shape: tuple[int, ...], M: int, reused_block_size: int | None = None
) -> dict[str, Any]:
"""Choose the shrink kernel config and the matching routing block size."""
N = weight_shape[1]
if M < 512:
config = {
"BLOCK_SIZE_M": 16,
"num_warps": 4 if (M <= 4 or N >= 32) else 2,
"num_stages": 3,
}
else:
config = {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2}
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
config = {"BLOCK_SIZE_M": 16, "num_warps": 4, "num_stages": 4}
if reused_block_size is not None:
config["BLOCK_SIZE_M"] = reused_block_size
return config
def _routing_cache_key(
num_experts, shared_outer, block_size, include_inactive_routes, adapter_enabled
):
return (
num_experts,
shared_outer,
block_size,
include_inactive_routes,
id(adapter_enabled),
)
def _align_virtual_topk_ids(
virtual_topk_ids, block_size, virtual_num_experts, populated_buckets
):
if virtual_num_experts < 1024:
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
moe_align_block_size as native_moe_align_block_size,
)
sorted_ids, expert_ids, count = native_moe_align_block_size(
virtual_topk_ids, block_size, virtual_num_experts
)
else:
sorted_ids, expert_ids, count = _align_block_size_large(
virtual_topk_ids, block_size, virtual_num_experts
)
num_tokens = virtual_topk_ids.numel()
max_nonempty = min(num_tokens, populated_buckets)
tight_padded = (
triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size)
* block_size
)
return sorted_ids[:tight_padded], expert_ids[: tight_padded // block_size], count
def _prewarm_per_expert_lora_routes(
hidden_states: torch.Tensor,
lora_a: torch.Tensor,
lora_b: torch.Tensor,
topk_ids: torch.Tensor,
token_lora_mapping: torch.Tensor,
routing_cache: dict,
*,
shrink_lora_b: torch.Tensor,
adapter_enabled: torch.Tensor | None = None,
include_inactive_routes: bool = False,
local_expert_offset: int = 0,
local_num_experts: int | None = None,
) -> bool:
"""Prepare paired per-expert stages on main and release raw IDs before returning.
The aligned cache retains only tables and their shared token mask. Global-ID
EP routing keeps the existing provider admission and staged fallback.
"""
if lora_a.shape[:2] != lora_b.shape[:2]:
return False
slots, num_experts = lora_a.shape[:2]
if local_num_experts is not None and local_num_experts < num_experts:
return False
if adapter_enabled is not None:
assert adapter_enabled.shape == (slots,)
assert adapter_enabled.dtype == torch.int32
assert adapter_enabled.device == topk_ids.device
if topk_ids.numel() == 0:
return True
a_shape = (slots * num_experts, lora_a.shape[2], lora_a.shape[3])
b_shape = (slots * num_experts, lora_b.shape[2], lora_b.shape[3])
tokens = token_lora_mapping.shape[0]
b_config = _get_stage_config(b_shape, 1, hidden_states.dtype, tokens)
reuse_b_block = (
tokens >= 512 and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
)
reused_block_size = None
if reuse_b_block:
shrink_b_shape = (
shrink_lora_b.shape[0] * shrink_lora_b.shape[1],
shrink_lora_b.shape[2],
shrink_lora_b.shape[3],
)
reused_block_size = _get_stage_config(
shrink_b_shape, 1, hidden_states.dtype, tokens
)["BLOCK_SIZE_M"]
a_config = _get_shrink_stage_config(a_shape, tokens, reused_block_size)
missing = {}
for block_size in (b_config["BLOCK_SIZE_M"], a_config["BLOCK_SIZE_M"]):
key = _routing_cache_key(
num_experts, False, block_size, include_inactive_routes, adapter_enabled
)
if key not in routing_cache:
missing[key] = block_size
if not missing:
return True
virtual_ids, token_mask, virtual_experts = _fused_virtual_topk_ids(
topk_ids,
token_lora_mapping,
num_experts,
False,
slots,
local_expert_offset,
local_num_experts,
adapter_enabled=adapter_enabled,
)
for key, block_size in missing.items():
sorted_ids, expert_ids, count = _align_virtual_topk_ids(
virtual_ids, block_size, virtual_experts, virtual_experts
)
routing_cache[key] = (sorted_ids, expert_ids, count, token_mask)
# Only aligned tables/masks survive the main-stream preparation scope.
del virtual_ids
return True
def _merged_experts_fused_moe_lora_add_impl(
output: torch.Tensor,
hidden_states: torch.Tensor,
@@ -441,6 +575,8 @@ def _merged_experts_fused_moe_lora_add_impl(
prewarm_a_routing: bool = True,
prewarm_b_routing: bool = True,
zero_intermediate: bool = False,
include_inactive_routes: bool = False,
adapter_enabled: torch.Tensor | None = None,
) -> "torch.Tensor | None":
"""
1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts.
@@ -462,6 +598,10 @@ def _merged_experts_fused_moe_lora_add_impl(
``broadcast_intermediate`` is an expand-only mode where one rank vector per
token is reused for every routed expert.
``include_inactive_routes`` retains negative routing blocks. The existing
expand then writes literal zero to private overwrite outputs, while ADD
projections preserve their inactive rows. Default routing is unchanged.
EP accepts either global weights/IDs with a local range or already-localized
weights/IDs from the standard dispatcher.
"""
@@ -470,82 +610,17 @@ def _merged_experts_fused_moe_lora_add_impl(
# dim 1) while lora_b is per-expert, so take the max for the true global count.
per_expert_dim = max(lora_a.shape[1], lora_b.shape[1])
ep_local = local_num_experts is not None and local_num_experts < per_expert_dim
input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1]
num_routes = topk_ids.numel()
input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1]
if adapter_enabled is not None:
assert adapter_enabled.shape == (max_loras,)
assert adapter_enabled.dtype == torch.int32
assert adapter_enabled.device == topk_ids.device
def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor:
# [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y]
return t.reshape(t.shape[0] * t.shape[1], t.shape[2], t.shape[3])
def _get_stage_config(
weight: torch.Tensor,
stage_top_k: int,
) -> dict[str, Any]:
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import (
get_config_dtype_str,
try_get_optimal_moe_config,
)
config_dtype = get_config_dtype_str(dtype=hidden_states.dtype)
get_config_func = functools.partial(
try_get_optimal_moe_config,
weight.shape,
weight.shape,
stage_top_k,
config_dtype,
)
try:
cfg = get_config_func(token_lora_mapping.shape[0])
except ValueError:
K_dim = weight.shape[2]
N_dim = weight.shape[1]
if K_dim >= 1024:
default_block_k = 256
elif K_dim >= 64:
default_block_k = 64
else:
default_block_k = max(16, K_dim)
cfg = {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": min(64, max(16, N_dim)),
"BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)),
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 4,
}
return cfg
def _get_shrink_stage_config(weight: torch.Tensor, M: int) -> dict[str, Any]:
"""Heuristic launch config for the LoRA A (shrink) stage.
Only BLOCK_SIZE_M / num_warps / num_stages are chosen here (a simple
decode/prefill rule). SPLIT_K is intentionally NOT set: the launcher
derives it from the post-alignment grid, see _invoke_moe_lora_shrink_splitk.
M is the number of input tokens; N (= weight.shape[1]) is the LoRA rank.
"""
N = weight.shape[1]
if M < 512: # decode / small batch: pin a small block, latency-bound
return {
"BLOCK_SIZE_M": 16,
"num_warps": 4 if (M <= 4 or N >= 32) else 2,
"num_stages": 3,
}
return {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2} # prefill
def _align_block_size(
topk_ids: torch.Tensor,
block_size: int,
num_experts: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# The native align kernel consumes num_experts + 1 internally for its
# sentinel bucket, so the 1024-expert boundary must use the fallback path.
if num_experts < 1024:
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
moe_align_block_size as native_moe_align_block_size,
)
return native_moe_align_block_size(topk_ids, block_size, num_experts)
return _align_block_size_large(topk_ids, block_size, num_experts)
def _get_routing(
topk_ids: torch.Tensor,
token_lora_mapping: torch.Tensor,
@@ -554,7 +629,13 @@ def _merged_experts_fused_moe_lora_add_impl(
block_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Check routing_cache for cross-call reuse (gate_up and down share routing)
cache_key = (num_experts, shared_outer, block_size)
cache_key = _routing_cache_key(
num_experts,
shared_outer,
block_size,
include_inactive_routes,
adapter_enabled,
)
if routing_cache is not None:
cached = routing_cache.get(cache_key)
if cached is not None:
@@ -593,8 +674,9 @@ def _merged_experts_fused_moe_lora_add_impl(
block_size,
local_expert_offset,
local_num_experts,
do_skip=True,
do_skip=not include_inactive_routes,
compact=compact_merged,
adapter_enabled=adapter_enabled,
)
result = (
sorted_token_ids,
@@ -615,36 +697,16 @@ def _merged_experts_fused_moe_lora_add_impl(
max_loras,
local_expert_offset,
local_num_experts,
adapter_enabled=adapter_enabled,
)
)
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size(
virtual_topk_ids,
block_size=block_size,
num_experts=virtual_num_experts,
)
# _align_block_size uses a worst-case padded allocation. Trim the routing buffers
# to a tighter upper bound so we keep the real routed work but drop unused padding.
# Under EP only this rank's experts are populated, so the per-expert stage bounds
# the non-empty buckets by `local_num_experts * max_loras` owned (expert, lora)
# combos + 1 sentinel bucket (all non-owned -> -1). The +1 matters: each non-empty
# bucket adds up to block-1 padding, so omitting it could truncate real tokens.
num_tokens = topk_ids.numel()
if ep_local and not shared_outer and local_num_experts < num_experts:
populated_buckets = local_num_experts * max_loras + 1
else:
populated_buckets = virtual_num_experts
max_nonempty = min(num_tokens, populated_buckets)
tight_padded = (
triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size)
* block_size
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_virtual_topk_ids(
virtual_topk_ids, block_size, virtual_num_experts, populated_buckets
)
sorted_token_ids = sorted_token_ids[:tight_padded]
expert_ids = expert_ids[: tight_padded // block_size]
# Single adapter (max_loras == 1): the per-adapter virtual-expert offset
# (safe_lora * num_experts) is always 0, so align never emits an id >=
# virtual_num_experts and fused_sanitize_expert_ids is an identity -> skip it.
if max_loras != 1:
expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts)
result = (
sorted_token_ids,
expert_ids,
@@ -673,11 +735,42 @@ def _merged_experts_fused_moe_lora_add_impl(
assert intermediate_buffer is not None
assert intermediate_buffer.ndim == 2
assert intermediate_buffer.shape[0] == token_lora_mapping.shape[0]
lora_a_virtual = _merge_lora_expert_weight(lora_a)
lora_b_virtual = _merge_lora_expert_weight(lora_b)
if num_routes == 0:
if stage == "shrink":
if intermediate_buffer is not None:
return intermediate_buffer
return hidden_states.new_empty(
(token_lora_mapping.shape[0], topk_ids.shape[1], max_lora_rank)
)
if stage == "expand":
assert intermediate_buffer is not None, (
"stage='expand' requires intermediate_buffer"
)
return None
num_experts_a = lora_a.shape[1]
num_experts_b = lora_b.shape[1]
b_stage_config = _get_stage_config(lora_b_virtual, 1)
a_shape = (max_loras * num_experts_a, lora_a.shape[2], lora_a.shape[3])
b_shape = (lora_b.shape[0] * num_experts_b, lora_b.shape[2], lora_b.shape[3])
tokens = token_lora_mapping.shape[0]
needs_shrink_config = stage in ("all", "shrink") or (
stage == "routing" and prewarm_a_routing
)
reuse_b_block = (
needs_shrink_config
and tokens >= 512
and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
)
needs_b_config = (
stage in ("all", "expand")
or (prewarm_b_routing and (stage == "routing" or routing_cache is not None))
or reuse_b_block
)
b_stage_config = (
_get_stage_config(b_shape, 1, hidden_states.dtype, tokens)
if needs_b_config
else None
)
if stage == "routing":
# Pre-warm the routing cache on the CALLER'S (main) stream so the
@@ -687,20 +780,12 @@ def _merged_experts_fused_moe_lora_add_impl(
# allocator's stream tracking is disabled while capturing), corrupting
# replays. Routing needs only topk_ids + token_lora_mapping, which are
# both ready before the side-stream fork, so it can run on main.
a_cfg = _get_shrink_stage_config(lora_a_virtual, token_lora_mapping.shape[0])
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
a_cfg = {**a_cfg, "BLOCK_SIZE_M": 16}
# Match the actual shrink-stage override below. Without this, callers
# that admit prefill-shaped batches into a side stream prewarm block-32
# routing here, then miss the cache when shrink switches to the B-stage
# block size. The miss allocates routing buffers on the side stream
# during capture, violating the allocation guarantee of stage='routing'.
if (
lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
and token_lora_mapping.shape[0] >= 512
):
a_cfg["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"]
if prewarm_a_routing:
a_cfg = _get_shrink_stage_config(
a_shape,
tokens,
b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None,
)
_get_routing(
topk_ids,
token_lora_mapping,
@@ -709,6 +794,7 @@ def _merged_experts_fused_moe_lora_add_impl(
a_cfg["BLOCK_SIZE_M"],
)
if prewarm_b_routing:
assert b_stage_config is not None
_get_routing(
topk_ids,
token_lora_mapping,
@@ -721,22 +807,9 @@ def _merged_experts_fused_moe_lora_add_impl(
intermediate = intermediate_buffer
if stage != "expand":
a_stage_config = _get_shrink_stage_config(
lora_a_virtual, token_lora_mapping.shape[0]
a_shape, tokens, b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None
)
if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get():
# Test-only override; the launcher fixes N and K block sizes.
a_stage_config = {
**a_stage_config,
"BLOCK_SIZE_M": 16,
"num_warps": 4,
"num_stages": 4,
}
# Match routing block sizes so prefill stages can share cached alignment.
if (
lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get()
and token_lora_mapping.shape[0] >= 512
):
a_stage_config["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"]
lora_a_virtual = _merge_lora_expert_weight(lora_a)
(
sorted_token_ids,
expert_ids,
@@ -800,6 +873,7 @@ def _merged_experts_fused_moe_lora_add_impl(
# Pre-warm the routing-B cache on this (side) stream so the later "expand" stage
# launches no routing kernels — they overlap finalize together with the shrink.
if routing_cache is not None and prewarm_b_routing:
assert b_stage_config is not None
_get_routing(
topk_ids,
token_lora_mapping,
@@ -810,6 +884,8 @@ def _merged_experts_fused_moe_lora_add_impl(
return intermediate
assert intermediate is not None, "stage='expand' requires intermediate_buffer"
assert b_stage_config is not None
lora_b_virtual = _merge_lora_expert_weight(lora_b)
(
sorted_token_ids,
expert_ids,
@@ -946,6 +1022,8 @@ def merged_experts_fused_moe_lora_add(
prewarm_a_routing: bool = True,
prewarm_b_routing: bool = True,
zero_intermediate: bool = False,
include_inactive_routes: bool = False,
adapter_enabled: torch.Tensor | None = None,
) -> "torch.Tensor | None":
"""Public API: wraps the registered op with routing_cache support."""
return _merged_experts_fused_moe_lora_add_impl(
@@ -972,4 +1050,6 @@ def merged_experts_fused_moe_lora_add(
prewarm_a_routing=prewarm_a_routing,
prewarm_b_routing=prewarm_b_routing,
zero_intermediate=zero_intermediate,
include_inactive_routes=include_inactive_routes,
adapter_enabled=adapter_enabled,
)
@@ -20,22 +20,21 @@ def _fused_virtual_topk_ids_kernel(
token_lora_mapping_ptr,
virtual_topk_ids_ptr,
token_lora_mask_ptr,
adapter_enabled_ptr,
stride_enabled,
num_experts_for_weight: tl.constexpr,
SHARED_OUTER: tl.constexpr,
NUM_SLOTS: tl.constexpr,
HAS_ADAPTER_ENABLED: tl.constexpr,
M,
top_k: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""
Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel.
Map active slots and valid base experts to virtual weight indices.
For each (m, k):
lora_id = token_lora_mapping[m]
mask[m] = (lora_id >= 0)
safe_lora = max(lora_id, 0)
if shared_outer: (handled by num_experts_for_weight == 0 sentinel)
virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora)
else:
virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight
Invalid slots or base experts map to -1. Shared weights accept any
nonnegative base expert and use the adapter slot as the weight index.
"""
pid = tl.program_id(0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
@@ -46,16 +45,29 @@ def _fused_virtual_topk_ids_kernel(
# k = offs % top_k # not needed directly
lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0)
mask_val = lora_id >= 0
safe_lora = tl.maximum(lora_id, 0)
mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS)
if HAS_ADAPTER_ENABLED:
if NUM_SLOTS == 1:
mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0)
else:
mask_val = mask_val & (
tl.load(
adapter_enabled_ptr + lora_id * stride_enabled,
mask=valid & mask_val,
other=0,
)
!= 0
)
safe_lora = tl.where(mask_val, lora_id, 0)
base = tl.load(topk_ids_ptr + offs, mask=valid, other=0)
# Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after
# EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on
# a real virtual-expert slot belonging to another adapter and trigger OOB
# loads in downstream LoRA kernels.
shifted = base + safe_lora * num_experts_for_weight
result = tl.where(base < 0, base, shifted)
valid_base = base >= 0
if not SHARED_OUTER:
valid_base = valid_base & (base < num_experts_for_weight)
expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0)
shifted = expert + safe_lora * num_experts_for_weight
# All align providers use one sentinel bucket; borrowed route IDs stay intact.
result = tl.where(mask_val & valid_base, shifted, -1)
tl.store(virtual_topk_ids_ptr + offs, result, mask=valid)
# Write mask once per row (at first k position)
@@ -70,6 +82,7 @@ def _fused_virtual_topk_ids(
num_experts: int,
shared_outer: bool,
max_loras: int,
adapter_enabled: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, int]:
"""
Returns virtual topk_ids, token_lora_mask, and virtual_num_experts.
@@ -77,27 +90,27 @@ def _fused_virtual_topk_ids(
M, top_k = topk_ids.shape
device = topk_ids.device
if shared_outer:
num_experts_for_weight = 1
# For shared_outer, we need topk_ids to be zeros
zero_topk = torch.zeros_like(topk_ids)
input_topk = zero_topk
else:
num_experts_for_weight = num_experts
input_topk = topk_ids
num_experts_for_weight = 1 if shared_outer else num_experts
virtual_topk_ids = torch.empty_like(topk_ids)
token_lora_mask = torch.empty(M, dtype=torch.bool, device=device)
if M * top_k == 0:
return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras
BLOCK_SIZE = 1024
grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,)
_fused_virtual_topk_ids_kernel[grid](
input_topk,
topk_ids,
token_lora_mapping,
virtual_topk_ids,
token_lora_mask,
adapter_enabled,
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
num_experts_for_weight,
shared_outer,
max_loras,
adapter_enabled is not None,
M,
top_k,
BLOCK_SIZE,
@@ -107,48 +120,6 @@ def _fused_virtual_topk_ids(
return virtual_topk_ids, token_lora_mask, virtual_num_experts
@triton.jit
def _fused_sanitize_expert_ids_kernel(
expert_ids_ptr,
output_ptr,
num_virtual_experts,
N,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
valid = offs < N
eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0)
result = tl.where(eid < num_virtual_experts, eid, -1)
tl.store(output_ptr + offs, result, mask=valid)
def fused_sanitize_expert_ids(
expert_ids: torch.Tensor,
num_virtual_experts: int,
) -> torch.Tensor:
"""
Sanitize expert_ids by replacing values >= num_virtual_experts with -1.
Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1.
"""
N = expert_ids.numel()
output = torch.empty_like(expert_ids)
BLOCK_SIZE = 1024
grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,)
_fused_sanitize_expert_ids_kernel[grid](
expert_ids,
output,
num_virtual_experts,
N,
BLOCK_SIZE,
)
return output
@triton.jit
def _moe_lora_shrink_splitk_kernel(
# Pointers
@@ -187,6 +158,9 @@ def _moe_lora_shrink_splitk_kernel(
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
if pid_mn >= num_pid_m * num_pid_n:
return
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid_mn // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
@@ -194,9 +168,6 @@ def _moe_lora_shrink_splitk_kernel(
pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m)
pid_n = (pid_mn % num_pid_in_group) // group_size_m
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
return
# Token routing (same pattern as fused_moe_triton_kernels)
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
@@ -309,7 +280,7 @@ def _align_block_size_jit(
scratch=None,
*,
return_inactive_prefix_count: bool = False,
map_invalid_experts_to_sentinel: bool = False,
map_invalid_experts_to_sentinel: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""CUDA JIT align_block_size for num_experts > 1024 (up to 8191).
@@ -319,8 +290,8 @@ def _align_block_size_jit(
``scratch(numel)`` returns the int32 scratch buffer (default: a fresh
allocation); a caller under CUDA graphs passes a persistent buffer.
IDs must be certified in [-1, num_experts) unless
``map_invalid_experts_to_sentinel`` maps other signed IDs to bucket zero.
Out-of-range signed IDs share the sentinel bucket; opt out only for IDs
certified in [-1, num_experts).
"""
assert num_experts <= 8191, (
f"_align_block_size_jit supports at most 8191 experts "
@@ -544,6 +515,9 @@ def _merged_experts_fused_moe_lora_add_impl(
experts_shared_outer_loras_a: bool,
experts_shared_outer_loras_b: bool,
routing_cache: dict | None = None,
adapter_enabled: torch.Tensor | None = None,
*,
a_output_slices: int = 1,
) -> None:
"""
1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts.
@@ -551,8 +525,21 @@ def _merged_experts_fused_moe_lora_add_impl(
3. Run regular SGLang fused-MoE kernels for LoRA A and LoRA B.
4. Mask out tokens with token_lora_mapping == -1 on the add path.
"""
assert a_output_slices in (1, 2)
if a_output_slices == 2:
assert lora_a.shape[2] == 2 * lora_b.shape[3]
assert lora_b.shape[2] > 0 and lora_b.shape[2] % 2 == 0
assert output.shape[-1] == lora_b.shape[2]
max_loras, _, max_lora_rank, _ = lora_a.shape
input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1]
num_routes = topk_ids.numel()
input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1]
if adapter_enabled is not None:
assert adapter_enabled.shape == (max_loras,)
assert adapter_enabled.device == topk_ids.device
assert adapter_enabled.dtype == torch.int32
if num_routes == 0:
return
def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor:
# [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y]
@@ -619,7 +606,7 @@ def _merged_experts_fused_moe_lora_add_impl(
block_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Check routing_cache for cross-call reuse (gate_up and down share routing)
cache_key = (num_experts, shared_outer, block_size)
cache_key = (num_experts, shared_outer, block_size, id(adapter_enabled))
if routing_cache is not None:
cached = routing_cache.get(cache_key)
if cached is not None:
@@ -627,7 +614,12 @@ def _merged_experts_fused_moe_lora_add_impl(
virtual_topk_ids, token_lora_mask, virtual_num_experts = (
_fused_virtual_topk_ids(
topk_ids, token_lora_mapping, num_experts, shared_outer, max_loras
topk_ids,
token_lora_mapping,
num_experts,
shared_outer,
max_loras,
adapter_enabled=adapter_enabled,
)
)
sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size(
@@ -645,7 +637,6 @@ def _merged_experts_fused_moe_lora_add_impl(
)
sorted_token_ids = sorted_token_ids[:tight_padded]
expert_ids = expert_ids[: tight_padded // block_size]
expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts)
result = (
sorted_token_ids,
expert_ids,
@@ -739,6 +730,7 @@ def _merged_experts_fused_moe_lora_add_impl(
fuse_add_to_output=True,
add_output_mask=token_lora_mask,
router_topk=topk_ids.shape[1],
a_output_slices=a_output_slices,
)
@@ -790,8 +782,11 @@ def merged_experts_fused_moe_lora_add(
experts_shared_outer_loras_a: bool,
experts_shared_outer_loras_b: bool,
routing_cache: dict | None = None,
adapter_enabled: torch.Tensor | None = None,
*,
a_output_slices: int = 1,
) -> None:
"""Public API: wraps the registered op with routing_cache support."""
"""Add virtual-expert LoRA deltas, optionally pairing A rank/output halves."""
_merged_experts_fused_moe_lora_add_impl(
output,
hidden_states,
@@ -804,4 +799,6 @@ def merged_experts_fused_moe_lora_add(
experts_shared_outer_loras_a,
experts_shared_outer_loras_b,
routing_cache,
adapter_enabled=adapter_enabled,
a_output_slices=a_output_slices,
)
@@ -382,6 +382,9 @@ def fused_moe_kernel(
FUSE_SWIGLU: tl.constexpr = False,
USE_GDC: tl.constexpr = False,
GDC_EARLY: tl.constexpr = False,
A_OUTPUT_SLICES: tl.constexpr = 1,
A_OUTPUT_SLICE_WIDTH: tl.constexpr = 0,
A_OUTPUT_SLICE_K: tl.constexpr = 0,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
@@ -420,7 +423,14 @@ def fused_moe_kernel(
# This is done in a grouped ordering to promote L2 data reuse.
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
if A_OUTPUT_SLICES == 2:
# Gate/up outputs use separate rank halves of the same shrink result.
# Keep each N tile inside one half, including ragged output widths.
slice_width: tl.constexpr = A_OUTPUT_SLICE_WIDTH
num_pid_n_per_slice: tl.constexpr = tl.cdiv(slice_width, BLOCK_SIZE_N)
num_pid_n = 2 * num_pid_n_per_slice
else:
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
@@ -445,7 +455,11 @@ def fused_moe_kernel(
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
off_experts = off_experts_i32.to(tl.int64)
if off_experts == -1:
if A_OUTPUT_SLICES == 2:
inactive_expert = off_experts < 0
else:
inactive_expert = off_experts == -1
if inactive_expert:
# The down GEMM never reads inactive sorted or half-width intermediates.
# Add/reduce outputs already hold their initial value and must stay intact.
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
@@ -463,7 +477,15 @@ def fused_moe_kernel(
)
return
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
if A_OUTPUT_SLICES == 2:
output_slice = pid_n // num_pid_n_per_slice
local_pid_n = pid_n % num_pid_n_per_slice
local_columns = local_pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(
tl.int64
)
offs_bn = local_columns % slice_width + output_slice * slice_width
else:
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
offs_k = tl.arange(0, BLOCK_SIZE_K)
if a_desc is not None:
assert use_fp8_w8a8 and group_n > 0 and group_k > 0
@@ -472,6 +494,8 @@ def fused_moe_kernel(
a_ptrs = a_ptr + (
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
)
if A_OUTPUT_SLICES == 2:
a_ptrs += output_slice * A_OUTPUT_SLICE_K * stride_ak
if b_desc is not None:
start_offs_n = pid_n * BLOCK_SIZE_N
@@ -610,7 +634,12 @@ def fused_moe_kernel(
accumulator = accumulator.to(compute_type)
# -----------------------------------------------------------
# Write back the block of the output
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
if A_OUTPUT_SLICES == 2:
offs_cn = local_columns + output_slice * slice_width
output_column_mask = local_columns < slice_width
else:
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
output_column_mask = offs_cn < N
if FUSE_SWIGLU:
# W13 rows were interleaved at load time, so gate/up of the same
@@ -667,7 +696,7 @@ def fused_moe_kernel(
offs_token_out = offs_token // ROUTER_TOPK
add_mask = tl.load(add_mask_ptr + offs_token_out, mask=token_mask, other=False)
c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :]
c_mask = token_mask[:, None] & add_mask[:, None] & (offs_cn[None, :] < N)
c_mask = token_mask[:, None] & add_mask[:, None] & output_column_mask[None, :]
existing = tl.load(c_ptrs, mask=c_mask, other=0.0)
tl.store(c_ptrs, existing + accumulator, mask=c_mask)
# ===== TO BE REFACTORED ====
@@ -810,10 +839,38 @@ def invoke_fused_moe_kernel(
mask_output: bool = False,
lora_preserve_base: bool = False,
fuse_swiglu: bool = False,
a_output_slices: int = 1,
) -> None:
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
"""Skip inactive experts, including with filter_expert=False.
a_output_slices=2 pairs consecutive A rank halves with B/C output halves
on the unquantized add path, without splitting the shrink/expand launches.
"""
assert topk_weights.stride(1) == 1
assert sorted_token_ids.stride(0) == 1
assert a_output_slices in (1, 2)
a_output_slice_width = 0
a_output_slice_k = 0
if a_output_slices == 2:
assert fuse_add_to_output
assert add_output_mask is not None
assert not mul_routed_weight
assert not (use_fp8_w8a8 or use_int8_w8a8 or use_int8_w8a16 or use_int4_w4a16)
assert not per_channel_quant and block_shape is None
assert not (a_use_tma or b_use_tma or fuse_swiglu or c_sorted)
assert not (
fuse_sum_all_reduce or mask_output or lora_preserve_base or no_combine
)
assert bias is None and A_scale is None and B_scale is None and B_zp is None
assert A.dtype in (torch.float16, torch.bfloat16)
assert B.dtype == C.dtype == A.dtype
assert compute_type == (tl.float16 if A.dtype == torch.float16 else tl.bfloat16)
assert A.ndim == 2 and B.ndim == 3
assert A.shape[1] == 2 * B.shape[2]
assert B.shape[1] > 0 and B.shape[1] % 2 == 0
assert C.shape[-1] == B.shape[1]
a_output_slice_width = B.shape[1] // 2
a_output_slice_k = B.shape[2]
if fuse_swiglu:
# The epilogue assumes an interleaved-gate/up bf16 up-GEMM writing a
@@ -887,7 +944,11 @@ def invoke_fused_moe_kernel(
grid = lambda META: (
triton.cdiv(sorted_token_ids.shape[0], META["BLOCK_SIZE_M"])
* triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]),
* (
triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"])
if a_output_slices == 1
else 2 * triton.cdiv(a_output_slice_width, META["BLOCK_SIZE_N"])
),
)
K = B.shape[2] - padded_size
@@ -1037,6 +1098,9 @@ def invoke_fused_moe_kernel(
c_sorted=c_sorted,
swap_ab=swap_ab,
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
A_OUTPUT_SLICES=a_output_slices,
A_OUTPUT_SLICE_WIDTH=a_output_slice_width,
A_OUTPUT_SLICE_K=a_output_slice_k,
MASK_OUTPUT=mask_output,
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
ROUTER_TOPK=router_topk,
@@ -72,7 +72,7 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
topk_weights = topk_output.topk_weights
assert runner_config.activation == "silu", "Only SiLU activation is supported."
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
@@ -96,7 +96,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
quant_info.global_num_experts if quant_info.expert_map is not None else E
)
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids, block_size_m, align_num_experts
topk_ids,
block_size_m,
align_num_experts,
return_inactive_prefix_count=quant_info.expert_map is None,
)
# Per-call workspace like fused_experts_none_to_marlin: a shared buffer aliases
@@ -119,13 +122,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
)
# Stage 1: Gate/Up (Marlin)
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -145,6 +144,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
top_k=topk,
mul_topk_weights=False,
is_ep=quant_info.expert_map is not None,
expert_ids_are_sorted=quant_info.expert_map is None,
has_inactive_prefix_count=quant_info.expert_map is None,
b_q_type=scalar_type1,
size_m=M,
size_n=2 * N,
@@ -169,6 +170,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: Down (Marlin)
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -188,7 +192,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
moe_block_size=block_size_m,
top_k=1,
mul_topk_weights=True,
zero_inactive_output=True,
is_ep=quant_info.expert_map is not None,
expert_ids_are_sorted=quant_info.expert_map is None,
has_inactive_prefix_count=quant_info.expert_map is None,
b_q_type=scalar_type2,
size_m=M * topk,
size_n=K,
+9 -1
View File
@@ -189,7 +189,12 @@ class LoRAInfo:
@dataclass
class LoRAHooks:
"""Hook callbacks for injecting LoRA deltas into the MoE pipeline."""
"""Hook callbacks for injecting LoRA deltas into the MoE pipeline.
Dropped route IDs remain negative. Hooks must skip those routes before
reading their payloads or scores; private inactive intermediates may be
uninitialized and routing scores are passed without a sanitized copy.
"""
after_gate_up: (
Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], None] | None
@@ -357,6 +362,8 @@ def _add_lora_gate_up_delta(
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
experts_shared_outer_loras_b=False,
routing_cache=routing_cache,
adapter_enabled=lora_info.adapter_enabled,
a_output_slices=2 if is_gated else 1,
)
else:
blk = _get_moe_lora_block_config(r)
@@ -432,6 +439,7 @@ def _add_lora_down_delta(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
routing_cache=routing_cache,
adapter_enabled=lora_info.adapter_enabled,
)
else:
blk = _get_moe_lora_block_config(lora_info.max_lora_rank)
@@ -14,11 +14,34 @@ import triton
import triton.language as tl
@triton.jit
def _silu_mul_add_delta_row(
x_ptr,
d_ptr,
out_ptr,
N,
row,
cols,
mask,
delta_mask,
):
x_row = x_ptr + row * 2 * N
d_row = d_ptr + row * 2 * N
gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32)
up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32)
gate += tl.load(d_row + cols, mask=delta_mask, other=0.0).to(tl.float32)
up += tl.load(d_row + N + cols, mask=delta_mask, other=0.0).to(tl.float32)
silu = gate * tl.sigmoid(gate)
out = silu * up
tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask)
@triton.jit
def _silu_mul_add_delta_kernel(
x_ptr, # [T, 2N] gemm1 output, [gate || up]
d_ptr, # [T, 2N] LoRA gate_up delta, same layout
out_ptr, # [T, N]
x_ptr,
d_ptr,
out_ptr,
N,
BLOCK: tl.constexpr,
):
@@ -26,17 +49,7 @@ def _silu_mul_add_delta_kernel(
block = tl.program_id(1)
cols = block * BLOCK + tl.arange(0, BLOCK)
mask = cols < N
x_row = x_ptr + row * 2 * N
d_row = d_ptr + row * 2 * N
gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32)
up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32)
gate += tl.load(d_row + cols, mask=mask, other=0.0).to(tl.float32)
up += tl.load(d_row + N + cols, mask=mask, other=0.0).to(tl.float32)
silu = gate * tl.sigmoid(gate)
out = silu * up
tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask)
_silu_mul_add_delta_row(x_ptr, d_ptr, out_ptr, N, row, cols, mask, mask)
def silu_and_mul_add_delta(
@@ -44,7 +57,7 @@ def silu_and_mul_add_delta(
gate_up_delta: torch.Tensor,
out: torch.Tensor,
) -> None:
"""``out = silu(g + dg) * (u + du)`` over contiguous ``[T, 2N]`` inputs."""
"""Both producers must define inactive route rows and disabled deltas as zero."""
T, two_n = gemm1_out.shape
N = two_n // 2
assert gate_up_delta.shape == (T, two_n) and out.shape == (T, N)
@@ -24,6 +24,7 @@ def _direct_gate_expand_kernel(
gate_b_ptr, # [S, E, 2I, R]
topk_ids_ptr, # [M, topk]
token_lora_mapping_ptr, # [M]
adapter_enabled_ptr,
output_ptr, # [M, topk, 2I]
stride_as,
stride_am,
@@ -34,11 +35,13 @@ def _direct_gate_expand_kernel(
stride_br,
stride_oq,
stride_on,
stride_enabled,
ROUTER_TOPK: tl.constexpr,
INTERMEDIATE_SIZE: tl.constexpr,
RANK: tl.constexpr,
GATE_WIDTH: tl.constexpr,
NUM_SLOTS: tl.constexpr,
HAS_ADAPTER_ENABLED: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
@@ -48,6 +51,15 @@ def _direct_gate_expand_kernel(
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
if HAS_ADAPTER_ENABLED:
active = active & (
tl.load(
adapter_enabled_ptr + slot * stride_enabled,
mask=(slot >= 0) & (slot < NUM_SLOTS),
other=0,
)
!= 0
)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
@@ -91,6 +103,7 @@ def _direct_down_shrink_kernel(
down_a_ptr, # [S, E, R, I]
topk_ids_ptr, # [M, topk]
token_lora_mapping_ptr, # [M]
adapter_enabled_ptr,
output_ptr, # [M, topk, R]
stride_xq,
stride_xk,
@@ -100,10 +113,12 @@ def _direct_down_shrink_kernel(
stride_ak,
stride_oq,
stride_or,
stride_enabled,
ROUTER_TOPK: tl.constexpr,
INTERMEDIATE_SIZE: tl.constexpr,
RANK: tl.constexpr,
NUM_SLOTS: tl.constexpr,
HAS_ADAPTER_ENABLED: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
):
@@ -112,6 +127,15 @@ def _direct_down_shrink_kernel(
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
if HAS_ADAPTER_ENABLED:
active = active & (
tl.load(
adapter_enabled_ptr + slot * stride_enabled,
mask=(slot >= 0) & (slot < NUM_SLOTS),
other=0,
)
!= 0
)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_r = tl.arange(0, RANK)
@@ -153,6 +177,8 @@ def direct_decode_gate_expand(
topk_ids: torch.Tensor,
token_lora_mapping: torch.Tensor,
output: torch.Tensor,
*,
adapter_enabled: torch.Tensor | None = None,
) -> None:
"""Expand shared gate/up rank vectors through directly selected experts.
@@ -163,6 +189,10 @@ def direct_decode_gate_expand(
"""
num_tokens = topk_ids.shape[0]
if adapter_enabled is not None:
assert adapter_enabled.shape == (gate_b.shape[0],)
assert adapter_enabled.device == topk_ids.device
assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64)
if shared_intermediate.ndim == 2:
shared_slot_stride = 0
shared_token_stride = shared_intermediate.stride(0)
@@ -180,6 +210,7 @@ def direct_decode_gate_expand(
gate_b,
topk_ids,
token_lora_mapping,
adapter_enabled,
output_flat,
shared_slot_stride,
shared_token_stride,
@@ -190,11 +221,13 @@ def direct_decode_gate_expand(
gate_b.stride(3),
output_flat.stride(0),
output_flat.stride(1),
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
ROUTER_TOPK=ROUTER_TOPK,
INTERMEDIATE_SIZE=intermediate_size,
RANK=RANK,
GATE_WIDTH=gate_width,
NUM_SLOTS=gate_b.shape[0],
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
BLOCK_M=_BLOCK_M,
BLOCK_N=_GATE_BLOCK_N,
num_warps=4,
@@ -208,6 +241,8 @@ def direct_decode_down_shrink(
topk_ids: torch.Tensor,
token_lora_mapping: torch.Tensor,
output: torch.Tensor,
*,
adapter_enabled: torch.Tensor | None = None,
) -> None:
"""Shrink directly routed activations through per-expert LoRA-A.
@@ -216,6 +251,10 @@ def direct_decode_down_shrink(
"""
num_tokens = topk_ids.shape[0]
if adapter_enabled is not None:
assert adapter_enabled.shape == (down_a.shape[0],)
assert adapter_enabled.device == topk_ids.device
assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64)
intermediate_size = activation.shape[1]
output_flat = output.view(num_tokens * ROUTER_TOPK, RANK)
num_routes = topk_ids.numel()
@@ -224,6 +263,7 @@ def direct_decode_down_shrink(
down_a,
topk_ids,
token_lora_mapping,
adapter_enabled,
output_flat,
activation.stride(0),
activation.stride(1),
@@ -233,10 +273,12 @@ def direct_decode_down_shrink(
down_a.stride(3),
output_flat.stride(0),
output_flat.stride(1),
adapter_enabled.stride(0) if adapter_enabled is not None else 0,
ROUTER_TOPK=ROUTER_TOPK,
INTERMEDIATE_SIZE=intermediate_size,
RANK=RANK,
NUM_SLOTS=down_a.shape[0],
HAS_ADAPTER_ENABLED=adapter_enabled is not None,
BLOCK_M=_BLOCK_M,
BLOCK_K=_DOWN_BLOCK_K,
num_warps=4,
@@ -209,6 +209,7 @@ def fused_experts_experimental_sgl_marlin_lora(
lora_info,
) -> StandardCombineInput:
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
_prewarm_per_expert_lora_routes,
merged_experts_fused_moe_lora_add,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
@@ -233,7 +234,7 @@ def fused_experts_experimental_sgl_marlin_lora(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
topk_weights = topk_output.topk_weights
if (
topk_ids.ndim != 2
or topk_weights.shape != topk_ids.shape
@@ -345,8 +346,25 @@ def fused_experts_experimental_sgl_marlin_lora(
!= lora_info.gate_up_lora_b_weights.shape[-1]
)
# Allocate and prewarm on main; side streams launch kernels only.
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
if not direct_decode:
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
paired_routing = (
factored_shared_outer
and not direct_decode
and _prewarm_per_expert_lora_routes(
hidden_states,
lora_info.down_lora_a_weights,
lora_info.gate_up_lora_b_weights,
topk_ids,
lora_info.token_lora_mapping,
routing_cache,
shrink_lora_b=lora_info.down_lora_b_weights,
adapter_enabled=lora_info.adapter_enabled,
include_inactive_routes=True,
local_expert_offset=0,
local_num_experts=E,
)
)
if not direct_decode and not paired_routing:
merged_experts_fused_moe_lora_add(
output=gate_up_delta,
hidden_states=hidden_states,
@@ -359,7 +377,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
experts_shared_outer_loras_b=False,
routing_cache=routing_cache,
include_inactive_routes=True,
stage="routing",
adapter_enabled=lora_info.adapter_enabled,
prewarm_a_routing=not factored_shared_outer,
local_expert_offset=0,
local_num_experts=E,
@@ -380,7 +400,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=True,
experts_shared_outer_loras_b=False,
routing_cache=collapsed_routing_cache,
include_inactive_routes=True,
stage="routing",
adapter_enabled=lora_info.adapter_enabled,
prewarm_a_routing=True,
prewarm_b_routing=False,
local_expert_offset=0,
@@ -402,7 +424,7 @@ def fused_experts_experimental_sgl_marlin_lora(
)
)
# Prewarm down routing and allocate its intermediate on main.
if not direct_decode:
if not direct_decode and not paired_routing:
merged_experts_fused_moe_lora_add(
output=gate_up_delta,
hidden_states=hidden_states,
@@ -415,8 +437,10 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
routing_cache=routing_cache,
include_inactive_routes=True,
stage="routing",
prewarm_b_routing=not factored_shared_outer,
adapter_enabled=lora_info.adapter_enabled,
local_expert_offset=0,
local_num_experts=E,
)
@@ -436,8 +460,10 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=True,
routing_cache=collapsed_routing_cache,
include_inactive_routes=True,
stage="routing",
prewarm_a_routing=False,
adapter_enabled=lora_info.adapter_enabled,
prewarm_b_routing=True,
local_expert_offset=0,
local_num_experts=E,
@@ -476,7 +502,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=True,
experts_shared_outer_loras_b=False,
routing_cache=collapsed_routing_cache,
include_inactive_routes=True,
stage="shrink",
adapter_enabled=lora_info.adapter_enabled,
prewarm_b_routing=False,
intermediate_buffer=gate_up_intermediate,
local_expert_offset=0,
@@ -495,6 +523,7 @@ def fused_experts_experimental_sgl_marlin_lora(
topk_ids,
lora_info.token_lora_mapping,
gate_up_delta,
adapter_enabled=lora_info.adapter_enabled,
)
return
merged_experts_fused_moe_lora_add(
@@ -509,7 +538,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=True,
experts_shared_outer_loras_b=False,
routing_cache=routing_cache,
include_inactive_routes=True,
fuse_add_to_output=False,
adapter_enabled=lora_info.adapter_enabled,
use_direct_expand_add=True,
local_expert_offset=0,
local_num_experts=E,
@@ -530,7 +561,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras,
experts_shared_outer_loras_b=False,
routing_cache=routing_cache,
include_inactive_routes=True,
fuse_add_to_output=False,
adapter_enabled=lora_info.adapter_enabled,
use_direct_expand_add=use_direct_expand,
local_expert_offset=0,
local_num_experts=E,
@@ -552,7 +585,11 @@ def fused_experts_experimental_sgl_marlin_lora(
# Start alignment after the side-stream fork. EP IDs are already localized;
# the wrapper owns Marlin's extra sentinel bucket.
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids, block_size_m, E
topk_ids,
block_size_m,
E,
return_inactive_prefix_count=not ep_active,
map_invalid_experts_to_sentinel=False,
)
# A cached workspace can outlive its CUDA graph-private pool.
@@ -572,13 +609,9 @@ def fused_experts_experimental_sgl_marlin_lora(
)
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -597,7 +630,10 @@ def fused_experts_experimental_sgl_marlin_lora(
moe_block_size=block_size_m,
top_k=topk,
mul_topk_weights=False,
zero_inactive_output=True,
is_ep=ep_active,
expert_ids_are_sorted=not ep_active,
has_inactive_prefix_count=not ep_active,
b_q_type=scalar_type1,
size_m=M,
size_n=2 * N,
@@ -616,6 +652,8 @@ def fused_experts_experimental_sgl_marlin_lora(
if run_lora:
if lora_event is not None:
torch.cuda.current_stream().wait_event(lora_event)
# Both private producers define dropped routes as zero; the runtime
# activation therefore needs no routing or adapter metadata reads.
silu_and_mul_add_delta(
intermediate_cache1.view(-1, 2 * N),
gate_up_delta.view(-1, 2 * N),
@@ -641,6 +679,7 @@ def fused_experts_experimental_sgl_marlin_lora(
topk_ids,
lora_info.token_lora_mapping,
down_intermediate,
adapter_enabled=lora_info.adapter_enabled,
)
else:
merged_experts_fused_moe_lora_add(
@@ -655,7 +694,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
routing_cache=routing_cache,
include_inactive_routes=True,
stage="shrink",
adapter_enabled=lora_info.adapter_enabled,
prewarm_b_routing=not factored_shared_outer,
intermediate_buffer=down_intermediate,
local_expert_offset=0,
@@ -673,6 +714,8 @@ def fused_experts_experimental_sgl_marlin_lora(
down_rank_sum,
routed_scaling_factor,
block_m=_weighted_rank_sum_block_m(M),
topk_ids=topk_ids,
inactive_ranks_are_zero=True,
)
down_shrink_done.record()
if torch.cuda.is_current_stream_capturing():
@@ -682,6 +725,9 @@ def fused_experts_experimental_sgl_marlin_lora(
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: down (marlin).
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -701,7 +747,10 @@ def fused_experts_experimental_sgl_marlin_lora(
moe_block_size=block_size_m,
top_k=1,
mul_topk_weights=True,
zero_inactive_output=True,
is_ep=ep_active,
expert_ids_are_sorted=not ep_active,
has_inactive_prefix_count=not ep_active,
b_q_type=scalar_type2,
size_m=M * topk,
size_n=K,
@@ -723,6 +772,7 @@ def fused_experts_experimental_sgl_marlin_lora(
topk_ids,
lora_info.token_lora_mapping,
down_intermediate,
adapter_enabled=lora_info.adapter_enabled,
)
else:
down_intermediate.zero_()
@@ -738,7 +788,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=True,
routing_cache=routing_cache,
include_inactive_routes=True,
stage="shrink",
adapter_enabled=lora_info.adapter_enabled,
prewarm_b_routing=False,
intermediate_buffer=down_intermediate,
local_expert_offset=0,
@@ -760,7 +812,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
routing_cache=routing_cache,
include_inactive_routes=True,
local_expert_offset=0,
adapter_enabled=lora_info.adapter_enabled,
local_num_experts=E,
zero_intermediate=lora_info.experts_shared_outer_loras,
)
@@ -782,6 +836,7 @@ def fused_experts_experimental_sgl_marlin_lora(
output,
routed_scaling_factor,
block_k=64,
topk_ids=topk_ids,
)
else:
tail_block_m, tail_block_k = fused_base_shared_lora_reduce_config(M)
@@ -794,6 +849,7 @@ def fused_experts_experimental_sgl_marlin_lora(
routed_scaling_factor,
block_m=tail_block_m,
block_k=tail_block_k,
topk_ids=topk_ids,
)
elif routed_scaling_factor == 1.0 and M <= 512:
torch.sum(intermediate_cache3, dim=1, out=output)
@@ -810,6 +866,8 @@ def fused_experts_experimental_sgl_marlin_lora(
down_rank_sum,
routed_scaling_factor,
block_m=_weighted_rank_sum_block_m(M),
topk_ids=topk_ids,
inactive_ranks_are_zero=True,
)
else:
# Wait only at the shared-B consumer.
@@ -830,7 +888,9 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=True,
routing_cache=collapsed_routing_cache,
include_inactive_routes=True,
fuse_add_to_output=True,
adapter_enabled=lora_info.adapter_enabled,
use_direct_expand_add=False,
local_expert_offset=0,
local_num_experts=E,
@@ -854,8 +914,10 @@ def fused_experts_experimental_sgl_marlin_lora(
experts_shared_outer_loras_a=False,
experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras,
routing_cache=routing_cache,
include_inactive_routes=True,
fuse_add_to_output=False,
fuse_sum_all_reduce=True,
adapter_enabled=lora_info.adapter_enabled,
use_direct_expand_add=lora_info.max_lora_rank <= 64,
local_expert_offset=0,
local_num_experts=E,
@@ -16,6 +16,7 @@ import triton.language as tl
def _weighted_topk_rank_sum_kernel(
routed_rank_ptr, # [M, topk, R]
topk_weights_ptr, # [M, topk]
topk_ids_ptr,
output_ptr, # [M, R]
M,
R,
@@ -25,9 +26,13 @@ def _weighted_topk_rank_sum_kernel(
stride_rr,
stride_wm,
stride_wk,
stride_im,
stride_ik,
stride_om,
stride_or,
TOPK: tl.constexpr,
HAS_TOPK_IDS: tl.constexpr,
INACTIVE_RANKS_ARE_ZERO: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_R: tl.constexpr,
):
@@ -40,12 +45,23 @@ def _weighted_topk_rank_sum_kernel(
accumulator = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32)
for k in tl.static_range(TOPK):
route_mask = token_mask
if HAS_TOPK_IDS:
route = tl.load(
topk_ids_ptr + token * stride_im + k * stride_ik,
mask=token_mask,
other=-1,
)
route_mask = route_mask & (route >= 0)
rank_load_mask = mask & route_mask[:, None]
if INACTIVE_RANKS_ARE_ZERO:
rank_load_mask = mask
routed_rank = tl.load(
routed_rank_ptr
+ token[:, None] * stride_rm
+ k * stride_rk
+ rank[None, :] * stride_rr,
mask=mask,
mask=rank_load_mask,
other=0.0,
).to(tl.float32)
weight = tl.load(
@@ -53,6 +69,8 @@ def _weighted_topk_rank_sum_kernel(
mask=token_mask,
other=0.0,
).to(tl.float32)
if HAS_TOPK_IDS:
weight = tl.where(route_mask, weight, 0.0)
accumulator += routed_rank * weight[:, None]
accumulator *= scale
@@ -69,18 +87,33 @@ def weighted_topk_rank_sum(
output: torch.Tensor,
routed_scaling_factor: float,
*,
topk_ids: torch.Tensor | None = None,
inactive_ranks_are_zero: bool = False,
block_m: int,
) -> None:
"""Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``."""
"""Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``.
``inactive_ranks_are_zero`` certifies that the rank producer writes literal
zero for every dropped route. Original IDs select inactive scores to zero
before multiplication, excluding NaNs from those scores. The default also
masks rank loads and admits poisoned private payloads.
"""
assert routed_rank.ndim == 3
assert topk_weights.shape == routed_rank.shape[:2]
assert output.shape == (routed_rank.shape[0], routed_rank.shape[2])
assert routed_rank.is_contiguous() and topk_weights.is_contiguous()
assert output.is_contiguous()
if inactive_ranks_are_zero:
assert topk_ids is not None
if routed_rank.shape[0] == 0:
return
if topk_ids is not None:
assert topk_ids.shape == topk_weights.shape
assert topk_ids.device == routed_rank.device
assert topk_ids.dtype in (torch.int32, torch.int64)
rank = routed_rank.shape[2]
assert block_m in (1, 2)
block_rank = max(16, triton.next_power_of_2(rank))
@@ -89,6 +122,7 @@ def weighted_topk_rank_sum(
](
routed_rank,
topk_weights,
topk_ids,
output,
routed_rank.shape[0],
rank,
@@ -98,9 +132,13 @@ def weighted_topk_rank_sum(
routed_rank.stride(2),
topk_weights.stride(0),
topk_weights.stride(1),
topk_ids.stride(0) if topk_ids is not None else 0,
topk_ids.stride(1) if topk_ids is not None else 0,
output.stride(0),
output.stride(1),
TOPK=routed_rank.shape[1],
HAS_TOPK_IDS=topk_ids is not None,
INACTIVE_RANKS_ARE_ZERO=inactive_ranks_are_zero,
BLOCK_M=block_m,
BLOCK_R=block_rank,
num_warps=1,
@@ -112,6 +150,7 @@ def _fused_base_shared_lora_reduce_kernel(
routed_base_ptr, # [M, topk, K], already weighted by the base MoE
routed_rank_ptr, # [M, topk, R]
topk_weights_ptr, # [M, topk]
topk_ids_ptr,
shared_b_ptr, # [K, R]
output_ptr, # [M, K]
M,
@@ -126,11 +165,14 @@ def _fused_base_shared_lora_reduce_kernel(
stride_rr,
stride_wm,
stride_wk,
stride_im,
stride_ik,
stride_sk,
stride_sr,
stride_om,
stride_ok,
TOPK: tl.constexpr,
HAS_TOPK_IDS: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_R: tl.constexpr,
@@ -147,12 +189,20 @@ def _fused_base_shared_lora_reduce_kernel(
base_acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
rank_acc = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32)
for topk_idx in tl.static_range(TOPK):
route_mask = token_mask
if HAS_TOPK_IDS:
route = tl.load(
topk_ids_ptr + token * stride_im + topk_idx * stride_ik,
mask=token_mask,
other=-1,
)
route_mask = route_mask & (route >= 0)
base = tl.load(
routed_base_ptr
+ token[:, None] * stride_bm
+ topk_idx * stride_bk
+ out_col[None, :] * stride_bn,
mask=token_mask[:, None] & out_mask[None, :],
mask=route_mask[:, None] & out_mask[None, :],
other=0.0,
).to(tl.float32)
routed_rank = tl.load(
@@ -160,12 +210,12 @@ def _fused_base_shared_lora_reduce_kernel(
+ token[:, None] * stride_rm
+ topk_idx * stride_rk
+ rank[None, :] * stride_rr,
mask=token_mask[:, None] & rank_mask[None, :],
mask=route_mask[:, None] & rank_mask[None, :],
other=0.0,
).to(tl.float32)
weight = tl.load(
topk_weights_ptr + token * stride_wm + topk_idx * stride_wk,
mask=token_mask,
mask=route_mask,
other=0.0,
).to(tl.float32)
base_acc += base
@@ -200,6 +250,7 @@ def fused_base_shared_lora_reduce(
output: torch.Tensor,
routed_scaling_factor: float,
*,
topk_ids: torch.Tensor | None = None,
block_m: int,
block_k: int,
) -> None:
@@ -237,6 +288,11 @@ def fused_base_shared_lora_reduce(
if routed_base.shape[0] == 0:
return
if topk_ids is not None:
assert topk_ids.shape == topk_weights.shape
assert topk_ids.device == routed_rank.device
assert topk_ids.dtype in (torch.int32, torch.int64)
rank = routed_rank.shape[2]
assert block_m in (1, 2, 4, 8)
assert block_k in (32, 64, 128)
@@ -250,6 +306,7 @@ def fused_base_shared_lora_reduce(
routed_base,
routed_rank,
topk_weights,
topk_ids,
shared_b,
output,
routed_base.shape[0],
@@ -264,11 +321,14 @@ def fused_base_shared_lora_reduce(
routed_rank.stride(2),
topk_weights.stride(0),
topk_weights.stride(1),
topk_ids.stride(0) if topk_ids is not None else 0,
topk_ids.stride(1) if topk_ids is not None else 0,
shared_b.stride(0),
shared_b.stride(1),
output.stride(0),
output.stride(1),
TOPK=routed_base.shape[1],
HAS_TOPK_IDS=topk_ids is not None,
BLOCK_M=block_m,
BLOCK_K=block_k,
BLOCK_R=block_rank,
@@ -284,6 +344,7 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
routed_base_ptr,
routed_rank_ptr,
topk_weights_ptr,
topk_ids_ptr,
shared_b_ptr, # [S, K, R]
token_lora_mapping_ptr,
output_ptr,
@@ -298,12 +359,15 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
stride_rr,
stride_wm,
stride_wk,
stride_im,
stride_ik,
stride_ss,
stride_sk,
stride_sr,
stride_om,
stride_ok,
TOPK: tl.constexpr,
HAS_TOPK_IDS: tl.constexpr,
NUM_SLOTS: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_R: tl.constexpr,
@@ -317,12 +381,16 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
base_acc = tl.zeros((1, BLOCK_K), dtype=tl.float32)
rank_acc = tl.zeros((1, BLOCK_R), dtype=tl.float32)
for topk_idx in tl.static_range(TOPK):
route_active = tl.full((), True, tl.int1)
if HAS_TOPK_IDS:
route = tl.load(topk_ids_ptr + token * stride_im + topk_idx * stride_ik)
route_active = route >= 0
base = tl.load(
routed_base_ptr
+ token * stride_bm
+ topk_idx * stride_bk
+ out_col[None, :] * stride_bn,
mask=out_mask[None, :],
mask=route_active & out_mask[None, :],
other=0.0,
).to(tl.float32)
routed_rank = tl.load(
@@ -330,10 +398,14 @@ def _fused_base_mapped_shared_lora_reduce_kernel(
+ token * stride_rm
+ topk_idx * stride_rk
+ rank[None, :] * stride_rr,
mask=rank_mask[None, :],
mask=route_active & rank_mask[None, :],
other=0.0,
).to(tl.float32)
weight = tl.load(topk_weights_ptr + token * stride_wm + topk_idx * stride_wk)
weight = tl.load(
topk_weights_ptr + token * stride_wm + topk_idx * stride_wk,
mask=route_active,
other=0.0,
)
base_acc += base
rank_acc += routed_rank * weight
@@ -366,6 +438,7 @@ def fused_base_mapped_shared_lora_reduce(
output: torch.Tensor,
routed_scaling_factor: float,
*,
topk_ids: torch.Tensor | None = None,
block_k: int,
) -> None:
"""Fused decode tail with a per-token shared-B adapter slot."""
@@ -417,6 +490,11 @@ def fused_base_mapped_shared_lora_reduce(
if routed_base.shape[0] == 0:
return
if topk_ids is not None:
assert topk_ids.shape == topk_weights.shape
assert topk_ids.device == routed_rank.device
assert topk_ids.dtype in (torch.int32, torch.int64)
assert block_k in (32, 64, 128)
block_rank = max(16, triton.next_power_of_2(routed_rank.shape[2]))
_fused_base_mapped_shared_lora_reduce_kernel[
@@ -425,6 +503,7 @@ def fused_base_mapped_shared_lora_reduce(
routed_base,
routed_rank,
topk_weights,
topk_ids,
shared_b_view,
token_lora_mapping,
output,
@@ -439,12 +518,15 @@ def fused_base_mapped_shared_lora_reduce(
routed_rank.stride(2),
topk_weights.stride(0),
topk_weights.stride(1),
topk_ids.stride(0) if topk_ids is not None else 0,
topk_ids.stride(1) if topk_ids is not None else 0,
shared_b_view.stride(0),
shared_b_view.stride(1),
shared_b_view.stride(2),
output.stride(0),
output.stride(1),
TOPK=routed_base.shape[1],
HAS_TOPK_IDS=topk_ids is not None,
NUM_SLOTS=shared_b.shape[0],
BLOCK_K=block_k,
BLOCK_R=block_rank,
@@ -65,6 +65,9 @@ def _moe_lora_expand_add_kernel(
num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
if pid >= num_pid_m * num_pid_n:
return
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
@@ -72,9 +75,6 @@ def _moe_lora_expand_add_kernel(
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
return
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
token_mask = offs_token < num_valid_tokens