From d0022e1df3fd71a439ad4cf58429e4ec0336b2f8 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:32:55 -0700 Subject: [PATCH] [Fix] lora: Preserve MoE projection identity and routing lifetimes (#45681) --- .../moe_lora_merged_align_kernel.cu | 151 ++- .../sglang/kernels/ops/lora/common/routing.py | 9 +- .../trtllm_lora_temp/moe_lora_merged_align.py | 35 +- .../moe/trtllm_lora_temp/virtual_experts.py | 516 ++++---- .../kernels/ops/lora/moe/virtual_experts.py | 155 ++- .../ops/moe/fused_moe_triton_kernels.py | 78 +- .../sglang/srt/lora/lora_moe_runner_marlin.py | 23 +- python/sglang/srt/lora/lora_moe_runners.py | 10 +- .../srt/lora/marlin_lora_temp/activation.py | 43 +- .../lora/marlin_lora_temp/direct_decode.py | 42 + .../srt/lora/marlin_lora_temp/moe_runner.py | 84 +- .../srt/lora/marlin_lora_temp/shared_outer.py | 98 +- .../trtllm_lora_temp/specialized_expand.py | 6 +- .../kernels/ops/lora/test_moe_lora_routing.py | 62 + .../kernels/ops/moe/test_moe_wna16_marlin.py | 134 ++- .../lora/test_virtual_experts_kernels.py | 1062 ++++++++++++++++- ...st_experimental_sgl_marlin_runtime_unit.py | 131 +- ...rimental_sgl_marlin_shared_outer_reduce.py | 405 ++++++- 18 files changed, 2561 insertions(+), 483 deletions(-) diff --git a/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu index 0bcfa0db58..f016cf854e 100644 --- a/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu +++ b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu @@ -48,11 +48,21 @@ using Vec = int4; namespace moe_lora_merged { +template +__device__ __forceinline__ bool +adapter_is_enabled(int lora_id, const int32_t* adapter_enabled, int num_slots, int64_t enabled_stride) { + if (lora_id < 0 || lora_id >= num_slots) return false; + if constexpr (HasAdapterEnabled) { + return adapter_enabled[lora_id * enabled_stride] != 0; + } + return true; +} + // Inline mirror of _fused_virtual_topk_ids_kernel (virtual_experts.py). Returns // the merged virtual expert id for flat slot `i` (range [-1, virtual_num_experts); // -1 is the dropped/masked sentinel). The caller adds +1 to get the histogram // bucket (sentinel -> bucket 0), matching the native +1 offset convention. -template +template __device__ __forceinline__ int compute_virtual_id( const scalar_t* __restrict__ topk_ids, const int32_t* __restrict__ token_lora_mapping, @@ -63,27 +73,32 @@ __device__ __forceinline__ int compute_virtual_id( int local_num_experts, bool ep_local, bool shared_outer, - bool compact) { + bool compact, + const int32_t* __restrict__ adapter_enabled, + int num_slots, + int64_t enabled_stride) { int m = static_cast(i) / top_k; int lora_id = token_lora_mapping[m]; - bool mask_val = lora_id >= 0; - int safe_lora = lora_id > 0 ? lora_id : 0; + bool mask_val = adapter_is_enabled(lora_id, adapter_enabled, num_slots, enabled_stride); - int base = shared_outer ? 0 : static_cast(topk_ids[i]); + scalar_t original_id = topk_ids[i]; + if (!mask_val || original_id < 0) return -1; + if (!shared_outer && original_id >= num_experts_for_weight) return -1; + int base = shared_outer ? 0 : static_cast(original_id); if (ep_local) { bool owned = base >= local_expert_offset && base < local_expert_offset + local_num_experts; base = owned ? base : -1; } - if (!mask_val || base < 0) return -1; + if (base < 0) return -1; // compact: dense LOCAL expert id in [0, local_num_experts) so the histogram // spans only local_num_experts buckets instead of the full global virtual - // space (337/385 empty under EP). Assumes max_loras==1 (safe_lora shift is 0; + // space (337/385 empty under EP). Assumes max_loras==1 (adapter shift is 0; // the wrapper guards). expert_ids is converted back to global at write time. if (compact) return base - local_expert_offset; - return base + safe_lora * num_experts_for_weight; + return base + lora_id * num_experts_for_weight; } -template +template __global__ void count_and_sort_expert_tokens_kernel( const scalar_t* __restrict__ topk_ids, const int32_t* __restrict__ token_lora_mapping, @@ -97,12 +112,15 @@ __global__ void count_and_sort_expert_tokens_kernel( bool ep_local, bool shared_outer, bool do_skip, - bool compact) { + bool compact, + const int32_t* __restrict__ adapter_enabled, + int num_slots, + int64_t enabled_stride) { const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; const size_t stride = blockDim.x * gridDim.x; for (size_t i = tid; i < numel; i += stride) { - int vid = compute_virtual_id( + int vid = compute_virtual_id( topk_ids, token_lora_mapping, i, @@ -112,7 +130,10 @@ __global__ void count_and_sort_expert_tokens_kernel( local_num_experts, ep_local, shared_outer, - compact); + compact, + adapter_enabled, + num_slots, + enabled_stride); // EP skip: dropped/masked slots (vid < 0) produce no delta on this rank, so // they never need a slot in sorted_token_ids -> skip the global atomicAdd // (kills the sentinel-bucket-0 contention). When do_skip is off they fall @@ -124,7 +145,7 @@ __global__ void count_and_sort_expert_tokens_kernel( } } -template +template __global__ void moe_align_block_size_kernel( const scalar_t* __restrict__ topk_ids, const int32_t* __restrict__ token_lora_mapping, @@ -146,7 +167,10 @@ __global__ void moe_align_block_size_kernel( bool ep_local, bool shared_outer, bool do_skip, - bool compact) { + bool compact, + const int32_t* __restrict__ adapter_enabled, + int num_slots, + int64_t enabled_stride) { // Use a separate thread block to populate sorted_token_ids if (blockIdx.x == 1) { if (pad_sorted_token_ids) { @@ -181,7 +205,7 @@ __global__ void moe_align_block_size_kernel( __syncthreads(); for (size_t i = tid; i < numel; i += stride) { - int vid = compute_virtual_id( + int vid = compute_virtual_id( topk_ids, token_lora_mapping, i, @@ -191,16 +215,20 @@ __global__ void moe_align_block_size_kernel( local_num_experts, ep_local, shared_outer, - compact); + compact, + adapter_enabled, + num_slots, + enabled_stride); // EP skip: dropped/masked slots don't increment any bucket (sentinel bucket // 0 stays empty), so they never get a block and never reach count_and_sort. if (!do_skip || vid >= 0) { atomicAdd(&shared_counts[vid + 1], 1); } - // token_lora_mask[m] = token_lora_mapping[m] >= 0, written once per row. + // Publish the same bounded adapter predicate used by both routing passes. if (static_cast(i) % top_k == 0) { int m = static_cast(i) / top_k; - token_lora_mask[m] = token_lora_mapping[m] >= 0; + token_lora_mask[m] = + adapter_is_enabled(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride); } } @@ -235,6 +263,9 @@ __global__ void moe_align_block_size_kernel( prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; s_total_tokens_post_pad = prefix[num_experts]; *total_tokens_post_pad = s_total_tokens_post_pad; + if constexpr (WriteInactivePrefixCount) { + total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size); + } } __syncthreads(); @@ -277,9 +308,9 @@ __global__ void moe_align_block_size_kernel( right = mid; } } - // compact buckets hold LOCAL expert ids; restore the global id (+offset) so - // the downstream GEMM still indexes the global contiguous LoRA weight. - expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0); + // Restore global IDs only for real compact buckets. The negative bucket + // stays -1 so overwrite consumers can zero it before touching weights. + expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0); } } @@ -287,9 +318,9 @@ __global__ void moe_align_block_size_kernel( // in ONE threadblock (one launch), eliminating the separate count_and_sort kernel // AND its redundant re-computation of the virtual id (cached in shared `svids`). // Only valid for small numel (the scatter is single-block); the wrapper routes -// large numel (prefill) to the 2-kernel path. do_skip is implied (this is the -// decode hot path); dropped slots are simply never scattered. -template +// large numel (prefill) to the 2-kernel path. do_skip omits dropped slots; +// overwrite consumers can retain them for the existing negative-expert store. +template __global__ void fused_align_scatter_kernel( const scalar_t* __restrict__ topk_ids, const int32_t* __restrict__ token_lora_mapping, @@ -310,7 +341,10 @@ __global__ void fused_align_scatter_kernel( bool ep_local, bool shared_outer, bool do_skip, - bool compact) { + bool compact, + const int32_t* __restrict__ adapter_enabled, + int num_slots, + int64_t enabled_stride) { extern __shared__ int32_t smem[]; int32_t* shared_counts = smem; // [num_experts] int32_t* prefix = shared_counts + num_experts; // [num_experts + 1] @@ -345,7 +379,7 @@ __global__ void fused_align_scatter_kernel( // Phase 2: histogram + cache the virtual id per slot + token_lora_mask. for (size_t i = tid; i < numel; i += stride) { - int vid = compute_virtual_id( + int vid = compute_virtual_id( topk_ids, token_lora_mapping, i, @@ -355,14 +389,18 @@ __global__ void fused_align_scatter_kernel( local_num_experts, ep_local, shared_outer, - compact); + compact, + adapter_enabled, + num_slots, + enabled_stride); svids[i] = vid; if (!do_skip || vid >= 0) { atomicAdd(&shared_counts[vid + 1], 1); } if (static_cast(i) % top_k == 0) { int m = static_cast(i) / top_k; - token_lora_mask[m] = token_lora_mapping[m] >= 0; + token_lora_mask[m] = + adapter_is_enabled(token_lora_mapping[m], adapter_enabled, num_slots, enabled_stride); } } __syncthreads(); @@ -387,6 +425,9 @@ __global__ void fused_align_scatter_kernel( prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; s_total_tokens_post_pad = prefix[num_experts]; *total_tokens_post_pad = s_total_tokens_post_pad; + if constexpr (WriteInactivePrefixCount) { + total_tokens_post_pad[1] = CEILDIV(shared_counts[0], block_size); + } } __syncthreads(); if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0; @@ -420,7 +461,7 @@ __global__ void fused_align_scatter_kernel( right = mid; } } - expert_ids[i] = left - 2 + (compact ? local_expert_offset : 0); + expert_ids[i] = left == 1 ? -1 : left - 2 + (compact ? local_expert_offset : 0); } if (tid < num_experts) cursor[tid] = prefix[tid]; __syncthreads(); @@ -437,7 +478,7 @@ __global__ void fused_align_scatter_kernel( } // namespace moe_lora_merged -template +template struct MoeLoraMergedAlignKernel { static void run(tvm::ffi::TensorView topk_ids, @@ -458,7 +499,10 @@ struct MoeLoraMergedAlignKernel { bool shared_outer, bool do_skip, bool compact, - bool fuse_scatter) { + bool fuse_scatter, + tvm::ffi::Optional adapter_enabled, + int64_t num_slots, + int64_t enabled_stride) { using namespace host; auto device = topk_ids.device(); @@ -475,12 +519,6 @@ struct MoeLoraMergedAlignKernel { // shared memory); the v2 (>1024) regime keeps the old path via the wrapper. RuntimeCheck( num_experts <= 1024, "moe_lora_merged_align: num_experts (bucket count) must be <= 1024, got ", num_experts); - // compact buckets hold LOCAL ids and restore global expert ids as - // (left-2+offset). For the sentinel bucket 0 that yields (offset-1), NOT the - // -1 the GEMM expects to skip -- only safe when do_skip empties bucket 0. - RuntimeCheck( - !compact || do_skip, "moe_lora_merged_align: compact requires do_skip (sentinel bucket must be empty)"); - const scalar_t* topk_ids_ptr = static_cast(topk_ids.data_ptr()); const int32_t* tlm_ptr = static_cast(token_lora_mapping.data_ptr()); bool* token_lora_mask_ptr = static_cast(token_lora_mask.data_ptr()); @@ -489,6 +527,21 @@ struct MoeLoraMergedAlignKernel { int32_t* num_tokens_post_pad_ptr = static_cast(num_tokens_post_pad.data_ptr()); int32_t* cumsum_buffer_ptr = static_cast(cumsum_buffer.data_ptr()); size_t numel = topk_ids.numel(); + const int32_t* enabled_ptr = nullptr; + if constexpr (HasAdapterEnabled) { + CHECK_HOST(adapter_enabled.has_value()); + TensorMatcher({num_slots}) + .with_dtype() + .with_device(device) + .with_strides({enabled_stride}) + .verify(adapter_enabled.value()); + enabled_ptr = static_cast(adapter_enabled.value().data_ptr()); + } + TensorMatcher({WriteInactivePrefixCount ? 2 : 1}) + .with_dtype() + .with_device(device) + .ensure_alignment(WriteInactivePrefixCount ? 8 : 4) + .verify(num_tokens_post_pad); const size_t scan_size = host::round_up_pow2(static_cast(num_experts)); @@ -497,7 +550,7 @@ struct MoeLoraMergedAlignKernel { // shared for the scatter cursor [num_experts] and cached virtual ids [numel]. const size_t shmem = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE + num_experts + numel) * sizeof(int32_t); - auto fused = moe_lora_merged::fused_align_scatter_kernel; + auto fused = moe_lora_merged::fused_align_scatter_kernel; LaunchKernel(dim3(1), dim3(threads), stream, shmem)( fused, topk_ids_ptr, @@ -519,13 +572,17 @@ struct MoeLoraMergedAlignKernel { ep_local, shared_outer, do_skip, - compact); + compact, + enabled_ptr, + static_cast(num_slots), + enabled_stride); return; } const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); - auto align_kernel = moe_lora_merged::moe_align_block_size_kernel; + auto align_kernel = + moe_lora_merged::moe_align_block_size_kernel; LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)( align_kernel, topk_ids_ptr, @@ -548,14 +605,19 @@ struct MoeLoraMergedAlignKernel { ep_local, shared_outer, do_skip, - compact); + compact, + enabled_ptr, + static_cast(num_slots), + enabled_stride); + + if (numel == 0) return; const int block_threads = std::min(256, threads); const int num_blocks = (numel + block_threads - 1) / block_threads; const int max_blocks = 65535; const int actual_blocks = std::min(num_blocks, max_blocks); - auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel; + auto sort_kernel = moe_lora_merged::count_and_sort_expert_tokens_kernel; LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( sort_kernel, topk_ids_ptr, @@ -570,7 +632,10 @@ struct MoeLoraMergedAlignKernel { ep_local, shared_outer, do_skip, - compact); + compact, + enabled_ptr, + static_cast(num_slots), + enabled_stride); } }; diff --git a/python/sglang/kernels/ops/lora/common/routing.py b/python/sglang/kernels/ops/lora/common/routing.py index 23b6c699e7..14e46dcee6 100644 --- a/python/sglang/kernels/ops/lora/common/routing.py +++ b/python/sglang/kernels/ops/lora/common/routing.py @@ -50,18 +50,17 @@ def route_bucket_ids( # Invalid slots/groups map to -1. With one group per slot, any nonnegative # group folds into that slot; otherwise bound it to the bucket table. token_ids = pair_ids // WIDTH - slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1).to(tl.int32) + slots = tl.load(token_slots_ptr + token_ids, mask=pair_mask, other=-1) live = (slots >= 0) & (slots < MAX_LORAS) + slots = slots.to(tl.int32) if HAS_GROUPS: - groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1).to( - tl.int32 - ) + groups = tl.load(group_ids_ptr + pair_ids, mask=pair_mask, other=-1) live = live & (groups >= 0) if GROUPS_PER_SLOT == 1: buckets = slots else: live = live & (groups < GROUPS_PER_SLOT) - buckets = slots * GROUPS_PER_SLOT + groups + buckets = slots * GROUPS_PER_SLOT + groups.to(tl.int32) else: buckets = slots return tl.where(live, buckets, -1) diff --git a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py index 183e77b5f1..d3bd8eb539 100644 --- a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py @@ -11,8 +11,16 @@ if TYPE_CHECKING: @cache_once -def _jit_module(dtype: torch.dtype) -> Module: - args = make_cpp_args(dtype) +def _jit_module( + dtype: torch.dtype, + has_adapter_enabled: bool = False, + write_inactive_prefix_count: bool = False, +) -> Module: + args = make_cpp_args( + dtype, + has_adapter_enabled, + write_inactive_prefix_count, + ) return load_jit( "moe_lora_merged_align", *args, @@ -35,6 +43,8 @@ def moe_lora_merged_align( do_skip: bool = True, compact: bool = False, fuse_scatter: Optional[bool] = None, + adapter_enabled: torch.Tensor | None = None, + return_inactive_prefix_count: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: """Fused replacement for (_fused_virtual_topk_ids + _align_block_size) on the merged-virtual-expert LoRA path. @@ -44,13 +54,17 @@ def moe_lora_merged_align( Returns (sorted_token_ids, expert_ids, num_tokens_post_padded, token_lora_mask, virtual_num_experts). + With ``return_inactive_prefix_count``, the count is an aligned int32 pair + [total padded tokens, leading negative-expert blocks]; do_skip publishes zero. """ device = topk_ids.device flat_topk_ids = topk_ids.reshape(-1) - if flat_topk_ids.dtype == torch.int64: - flat_topk_ids = flat_topk_ids.to(torch.int32) M, top_k = topk_ids.shape numel = M * top_k + if adapter_enabled is not None: + assert adapter_enabled.shape == (max_loras,) + assert adapter_enabled.dtype == torch.int32 + assert adapter_enabled.device == device num_experts_for_weight = 1 if shared_outer else num_experts virtual_num_experts = num_experts_for_weight * max_loras @@ -97,13 +111,19 @@ def moe_lora_merged_align( sorted_alloc = (max_num_tokens_padded + 3) & ~3 sorted_token_ids = torch.empty((sorted_alloc,), dtype=torch.int32, device=device) expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device) - num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device) + num_tokens_post_pad = torch.empty( + (2 if return_inactive_prefix_count else 1,), dtype=torch.int32, device=device + ) # No memset: the align kernel writes cumsum[0..num_buckets] before # count_and_sort reads it (same as moe_align_block_size.py's empty cumsum). cumsum_buffer = torch.empty((bucket_experts + 2,), dtype=torch.int32, device=device) token_lora_mask = torch.empty((M,), dtype=torch.bool, device=device) - module = _jit_module(flat_topk_ids.dtype) + module = _jit_module( + flat_topk_ids.dtype, + adapter_enabled is not None, + return_inactive_prefix_count, + ) module.moe_lora_merged_align( flat_topk_ids, token_lora_mapping, @@ -124,6 +144,9 @@ def moe_lora_merged_align( do_skip, compact_eff, fuse_eff, + adapter_enabled, + max_loras, + adapter_enabled.stride(0) if adapter_enabled is not None else 0, ) return ( diff --git a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py index 0836bccb3b..47b7260e38 100644 --- a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py @@ -22,26 +22,25 @@ def _fused_virtual_topk_ids_kernel( token_lora_mapping_ptr, virtual_topk_ids_ptr, token_lora_mask_ptr, + adapter_enabled_ptr, + stride_enabled, num_experts_for_weight: tl.constexpr, + SHARED_OUTER: tl.constexpr, M, top_k: tl.constexpr, BLOCK_SIZE: tl.constexpr, local_expert_offset: tl.constexpr, local_num_experts: tl.constexpr, EP_LOCAL: tl.constexpr, + NUM_SLOTS: tl.constexpr, + HAS_ADAPTER_ENABLED: tl.constexpr, ENABLE_PDL: tl.constexpr = False, ): """ - Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel. + Map active slots and valid base experts to virtual weight indices. - For each (m, k): - lora_id = token_lora_mapping[m] - mask[m] = (lora_id >= 0) - safe_lora = max(lora_id, 0) - if shared_outer: (handled by num_experts_for_weight == 0 sentinel) - virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora) - else: - virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight + Invalid slots or base experts map to -1. Shared weights accept any + nonnegative base expert and use the adapter slot as the weight index. """ pid = tl.program_id(0) offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) @@ -57,8 +56,20 @@ def _fused_virtual_topk_ids_kernel( # k = offs % top_k # not needed directly lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0) - mask_val = lora_id >= 0 - safe_lora = tl.maximum(lora_id, 0) + mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS) + if HAS_ADAPTER_ENABLED: + if NUM_SLOTS == 1: + mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0) + else: + mask_val = mask_val & ( + tl.load( + adapter_enabled_ptr + lora_id * stride_enabled, + mask=valid & mask_val, + other=0, + ) + != 0 + ) + safe_lora = tl.where(mask_val, lora_id, 0) base = tl.load(topk_ids_ptr + offs, mask=valid, other=0) if EP_LOCAL: @@ -70,13 +81,13 @@ def _fused_virtual_topk_ids_kernel( base < local_expert_offset + local_num_experts ) base = tl.where(owned, base, -1) - # Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after - # EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on - # a real virtual-expert slot belonging to another adapter and trigger OOB - # loads in downstream LoRA kernels. - shifted = base + safe_lora * num_experts_for_weight - result = tl.where(base < 0, base, shifted) - result = tl.where(mask_val, result, -1) + valid_base = base >= 0 + if not SHARED_OUTER: + valid_base = valid_base & (base < num_experts_for_weight) + expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0) + shifted = expert + safe_lora * num_experts_for_weight + # All align providers use one sentinel bucket; borrowed route IDs stay intact. + result = tl.where(mask_val & valid_base, shifted, -1) tl.store(virtual_topk_ids_ptr + offs, result, mask=valid) # Write mask once per row (at first k position) @@ -97,48 +108,52 @@ def _fused_virtual_topk_ids( max_loras: int, local_expert_offset: int = 0, local_num_experts: int | None = None, + adapter_enabled: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, int]: """ Returns virtual topk_ids, token_lora_mask, and virtual_num_experts. `local_num_experts` (< `num_experts`) enables the EP mask on the per-expert path: non-owned `topk_ids` slots are dropped to the -1 sentinel in-kernel. - The shared-outer path routes by lora id, not expert, so it is never masked - (else non-owned-but-valid tokens would be dropped on ranks > 0). + The shared-outer path drops negative routes but does not apply the local + expert range (its weights are shared by every owned expert). """ M, top_k = topk_ids.shape device = topk_ids.device - if shared_outer: - num_experts_for_weight = 1 - # For shared_outer, we need topk_ids to be zeros - zero_topk = torch.zeros_like(topk_ids) - input_topk = zero_topk - ep_local = False - else: - num_experts_for_weight = num_experts - input_topk = topk_ids - ep_local = local_num_experts is not None and local_num_experts < num_experts + num_experts_for_weight = 1 if shared_outer else num_experts + ep_local = ( + not shared_outer + and local_num_experts is not None + and local_num_experts < num_experts + ) virtual_topk_ids = torch.empty_like(topk_ids) token_lora_mask = torch.empty(M, dtype=torch.bool, device=device) + if M * top_k == 0: + return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras BLOCK_SIZE = 1024 grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,) enable_pdl, pdl_kwargs = get_pdl_launch_metadata() _fused_virtual_topk_ids_kernel[grid]( - input_topk, + topk_ids, token_lora_mapping, virtual_topk_ids, token_lora_mask, + adapter_enabled, + adapter_enabled.stride(0) if adapter_enabled is not None else 0, num_experts_for_weight, + shared_outer, M, top_k, BLOCK_SIZE, local_expert_offset, local_num_experts if local_num_experts is not None else 0, ep_local, + NUM_SLOTS=max_loras, + HAS_ADAPTER_ENABLED=adapter_enabled is not None, ENABLE_PDL=enable_pdl, **pdl_kwargs, ) @@ -147,61 +162,6 @@ def _fused_virtual_topk_ids( return virtual_topk_ids, token_lora_mask, virtual_num_experts -@triton.jit -def _fused_sanitize_expert_ids_kernel( - expert_ids_ptr, - output_ptr, - num_virtual_experts, - N, - BLOCK_SIZE: tl.constexpr, - ENABLE_PDL: tl.constexpr = False, -): - pid = tl.program_id(0) - offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - valid = offs < N - - # GDC wait: expert_ids comes from the immediately preceding align kernel; - # wait before consuming it. - if ENABLE_PDL: - tl.extra.cuda.gdc_wait() - - eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0) - result = tl.where(eid < num_virtual_experts, eid, -1) - tl.store(output_ptr + offs, result, mask=valid) - - # All work is done; hint the runtime to launch the dependent kernel. - if ENABLE_PDL: - tl.extra.cuda.gdc_launch_dependents() - - -def fused_sanitize_expert_ids( - expert_ids: torch.Tensor, - num_virtual_experts: int, -) -> torch.Tensor: - """ - Sanitize expert_ids by replacing values >= num_virtual_experts with -1. - - Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1. - """ - N = expert_ids.numel() - output = torch.empty_like(expert_ids) - - BLOCK_SIZE = 1024 - grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,) - - enable_pdl, pdl_kwargs = get_pdl_launch_metadata() - _fused_sanitize_expert_ids_kernel[grid]( - expert_ids, - output, - num_virtual_experts, - N, - BLOCK_SIZE, - ENABLE_PDL=enable_pdl, - **pdl_kwargs, - ) - return output - - @triton.jit def _moe_lora_shrink_splitk_kernel( # Pointers @@ -238,8 +198,8 @@ def _moe_lora_shrink_splitk_kernel( pid_mn = pid // SPLIT_K # GDC wait: the routing buffers (sorted_token_ids/expert_ids/ - # num_tokens_post_padded) come from the immediately preceding align and - # sanitize kernels; wait before consuming them. + # num_tokens_post_padded) come from routing preparation; wait before + # consuming them. if ENABLE_PDL: tl.extra.cuda.gdc_wait() @@ -247,6 +207,9 @@ def _moe_lora_shrink_splitk_kernel( num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M) num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + if pid_mn >= num_pid_m * num_pid_n: + return + num_pid_in_group = GROUP_SIZE_M * num_pid_n group_id = pid_mn // num_pid_in_group first_pid_m = group_id * GROUP_SIZE_M @@ -254,9 +217,6 @@ def _moe_lora_shrink_splitk_kernel( pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m) pid_n = (pid_mn % num_pid_in_group) // group_size_m - if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: - return - # Token routing (same pattern as fused_moe_triton_kernels) offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64) @@ -417,6 +377,180 @@ def _merged_experts_fused_moe_lora_add_fake( return +def _get_stage_config( + weight_shape: tuple[int, ...], + stage_top_k: int, + dtype: torch.dtype, + num_tokens: int, +) -> dict[str, Any]: + from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import ( + get_config_dtype_str, + try_get_optimal_moe_config, + ) + + config_dtype = get_config_dtype_str(dtype=dtype) + get_config_func = functools.partial( + try_get_optimal_moe_config, + weight_shape, + weight_shape, + stage_top_k, + config_dtype, + ) + try: + cfg = get_config_func(num_tokens) + except ValueError: + K_dim = weight_shape[2] + N_dim = weight_shape[1] + if K_dim >= 1024: + default_block_k = 256 + elif K_dim >= 64: + default_block_k = 64 + else: + default_block_k = max(16, K_dim) + cfg = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": min(64, max(16, N_dim)), + "BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)), + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4, + } + return cfg + + +def _get_shrink_stage_config( + weight_shape: tuple[int, ...], M: int, reused_block_size: int | None = None +) -> dict[str, Any]: + """Choose the shrink kernel config and the matching routing block size.""" + N = weight_shape[1] + if M < 512: + config = { + "BLOCK_SIZE_M": 16, + "num_warps": 4 if (M <= 4 or N >= 32) else 2, + "num_stages": 3, + } + else: + config = {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2} + if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get(): + config = {"BLOCK_SIZE_M": 16, "num_warps": 4, "num_stages": 4} + if reused_block_size is not None: + config["BLOCK_SIZE_M"] = reused_block_size + return config + + +def _routing_cache_key( + num_experts, shared_outer, block_size, include_inactive_routes, adapter_enabled +): + return ( + num_experts, + shared_outer, + block_size, + include_inactive_routes, + id(adapter_enabled), + ) + + +def _align_virtual_topk_ids( + virtual_topk_ids, block_size, virtual_num_experts, populated_buckets +): + if virtual_num_experts < 1024: + from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import ( + moe_align_block_size as native_moe_align_block_size, + ) + + sorted_ids, expert_ids, count = native_moe_align_block_size( + virtual_topk_ids, block_size, virtual_num_experts + ) + else: + sorted_ids, expert_ids, count = _align_block_size_large( + virtual_topk_ids, block_size, virtual_num_experts + ) + num_tokens = virtual_topk_ids.numel() + max_nonempty = min(num_tokens, populated_buckets) + tight_padded = ( + triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size) + * block_size + ) + return sorted_ids[:tight_padded], expert_ids[: tight_padded // block_size], count + + +def _prewarm_per_expert_lora_routes( + hidden_states: torch.Tensor, + lora_a: torch.Tensor, + lora_b: torch.Tensor, + topk_ids: torch.Tensor, + token_lora_mapping: torch.Tensor, + routing_cache: dict, + *, + shrink_lora_b: torch.Tensor, + adapter_enabled: torch.Tensor | None = None, + include_inactive_routes: bool = False, + local_expert_offset: int = 0, + local_num_experts: int | None = None, +) -> bool: + """Prepare paired per-expert stages on main and release raw IDs before returning. + + The aligned cache retains only tables and their shared token mask. Global-ID + EP routing keeps the existing provider admission and staged fallback. + """ + if lora_a.shape[:2] != lora_b.shape[:2]: + return False + slots, num_experts = lora_a.shape[:2] + if local_num_experts is not None and local_num_experts < num_experts: + return False + if adapter_enabled is not None: + assert adapter_enabled.shape == (slots,) + assert adapter_enabled.dtype == torch.int32 + assert adapter_enabled.device == topk_ids.device + if topk_ids.numel() == 0: + return True + a_shape = (slots * num_experts, lora_a.shape[2], lora_a.shape[3]) + b_shape = (slots * num_experts, lora_b.shape[2], lora_b.shape[3]) + tokens = token_lora_mapping.shape[0] + b_config = _get_stage_config(b_shape, 1, hidden_states.dtype, tokens) + reuse_b_block = ( + tokens >= 512 and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get() + ) + reused_block_size = None + if reuse_b_block: + shrink_b_shape = ( + shrink_lora_b.shape[0] * shrink_lora_b.shape[1], + shrink_lora_b.shape[2], + shrink_lora_b.shape[3], + ) + reused_block_size = _get_stage_config( + shrink_b_shape, 1, hidden_states.dtype, tokens + )["BLOCK_SIZE_M"] + a_config = _get_shrink_stage_config(a_shape, tokens, reused_block_size) + missing = {} + for block_size in (b_config["BLOCK_SIZE_M"], a_config["BLOCK_SIZE_M"]): + key = _routing_cache_key( + num_experts, False, block_size, include_inactive_routes, adapter_enabled + ) + if key not in routing_cache: + missing[key] = block_size + if not missing: + return True + virtual_ids, token_mask, virtual_experts = _fused_virtual_topk_ids( + topk_ids, + token_lora_mapping, + num_experts, + False, + slots, + local_expert_offset, + local_num_experts, + adapter_enabled=adapter_enabled, + ) + for key, block_size in missing.items(): + sorted_ids, expert_ids, count = _align_virtual_topk_ids( + virtual_ids, block_size, virtual_experts, virtual_experts + ) + routing_cache[key] = (sorted_ids, expert_ids, count, token_mask) + # Only aligned tables/masks survive the main-stream preparation scope. + del virtual_ids + return True + + def _merged_experts_fused_moe_lora_add_impl( output: torch.Tensor, hidden_states: torch.Tensor, @@ -441,6 +575,8 @@ def _merged_experts_fused_moe_lora_add_impl( prewarm_a_routing: bool = True, prewarm_b_routing: bool = True, zero_intermediate: bool = False, + include_inactive_routes: bool = False, + adapter_enabled: torch.Tensor | None = None, ) -> "torch.Tensor | None": """ 1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts. @@ -462,6 +598,10 @@ def _merged_experts_fused_moe_lora_add_impl( ``broadcast_intermediate`` is an expand-only mode where one rank vector per token is reused for every routed expert. + ``include_inactive_routes`` retains negative routing blocks. The existing + expand then writes literal zero to private overwrite outputs, while ADD + projections preserve their inactive rows. Default routing is unchanged. + EP accepts either global weights/IDs with a local range or already-localized weights/IDs from the standard dispatcher. """ @@ -470,82 +610,17 @@ def _merged_experts_fused_moe_lora_add_impl( # dim 1) while lora_b is per-expert, so take the max for the true global count. per_expert_dim = max(lora_a.shape[1], lora_b.shape[1]) ep_local = local_num_experts is not None and local_num_experts < per_expert_dim - input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1] + num_routes = topk_ids.numel() + input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1] + if adapter_enabled is not None: + assert adapter_enabled.shape == (max_loras,) + assert adapter_enabled.dtype == torch.int32 + assert adapter_enabled.device == topk_ids.device def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor: # [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y] return t.reshape(t.shape[0] * t.shape[1], t.shape[2], t.shape[3]) - def _get_stage_config( - weight: torch.Tensor, - stage_top_k: int, - ) -> dict[str, Any]: - from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe_triton_config import ( - get_config_dtype_str, - try_get_optimal_moe_config, - ) - - config_dtype = get_config_dtype_str(dtype=hidden_states.dtype) - get_config_func = functools.partial( - try_get_optimal_moe_config, - weight.shape, - weight.shape, - stage_top_k, - config_dtype, - ) - try: - cfg = get_config_func(token_lora_mapping.shape[0]) - except ValueError: - K_dim = weight.shape[2] - N_dim = weight.shape[1] - if K_dim >= 1024: - default_block_k = 256 - elif K_dim >= 64: - default_block_k = 64 - else: - default_block_k = max(16, K_dim) - cfg = { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": min(64, max(16, N_dim)), - "BLOCK_SIZE_K": min(default_block_k, max(16, K_dim)), - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4, - } - return cfg - - def _get_shrink_stage_config(weight: torch.Tensor, M: int) -> dict[str, Any]: - """Heuristic launch config for the LoRA A (shrink) stage. - - Only BLOCK_SIZE_M / num_warps / num_stages are chosen here (a simple - decode/prefill rule). SPLIT_K is intentionally NOT set: the launcher - derives it from the post-alignment grid, see _invoke_moe_lora_shrink_splitk. - M is the number of input tokens; N (= weight.shape[1]) is the LoRA rank. - """ - N = weight.shape[1] - if M < 512: # decode / small batch: pin a small block, latency-bound - return { - "BLOCK_SIZE_M": 16, - "num_warps": 4 if (M <= 4 or N >= 32) else 2, - "num_stages": 3, - } - return {"BLOCK_SIZE_M": 32, "num_warps": 2, "num_stages": 2} # prefill - - def _align_block_size( - topk_ids: torch.Tensor, - block_size: int, - num_experts: int, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - # The native align kernel consumes num_experts + 1 internally for its - # sentinel bucket, so the 1024-expert boundary must use the fallback path. - if num_experts < 1024: - from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import ( - moe_align_block_size as native_moe_align_block_size, - ) - - return native_moe_align_block_size(topk_ids, block_size, num_experts) - return _align_block_size_large(topk_ids, block_size, num_experts) - def _get_routing( topk_ids: torch.Tensor, token_lora_mapping: torch.Tensor, @@ -554,7 +629,13 @@ def _merged_experts_fused_moe_lora_add_impl( block_size: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: # Check routing_cache for cross-call reuse (gate_up and down share routing) - cache_key = (num_experts, shared_outer, block_size) + cache_key = _routing_cache_key( + num_experts, + shared_outer, + block_size, + include_inactive_routes, + adapter_enabled, + ) if routing_cache is not None: cached = routing_cache.get(cache_key) if cached is not None: @@ -593,8 +674,9 @@ def _merged_experts_fused_moe_lora_add_impl( block_size, local_expert_offset, local_num_experts, - do_skip=True, + do_skip=not include_inactive_routes, compact=compact_merged, + adapter_enabled=adapter_enabled, ) result = ( sorted_token_ids, @@ -615,36 +697,16 @@ def _merged_experts_fused_moe_lora_add_impl( max_loras, local_expert_offset, local_num_experts, + adapter_enabled=adapter_enabled, ) ) - sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size( - virtual_topk_ids, - block_size=block_size, - num_experts=virtual_num_experts, - ) - # _align_block_size uses a worst-case padded allocation. Trim the routing buffers - # to a tighter upper bound so we keep the real routed work but drop unused padding. - # Under EP only this rank's experts are populated, so the per-expert stage bounds - # the non-empty buckets by `local_num_experts * max_loras` owned (expert, lora) - # combos + 1 sentinel bucket (all non-owned -> -1). The +1 matters: each non-empty - # bucket adds up to block-1 padding, so omitting it could truncate real tokens. - num_tokens = topk_ids.numel() if ep_local and not shared_outer and local_num_experts < num_experts: populated_buckets = local_num_experts * max_loras + 1 else: populated_buckets = virtual_num_experts - max_nonempty = min(num_tokens, populated_buckets) - tight_padded = ( - triton.cdiv(num_tokens + max_nonempty * (block_size - 1), block_size) - * block_size + sorted_token_ids, expert_ids, num_tokens_post_padded = _align_virtual_topk_ids( + virtual_topk_ids, block_size, virtual_num_experts, populated_buckets ) - sorted_token_ids = sorted_token_ids[:tight_padded] - expert_ids = expert_ids[: tight_padded // block_size] - # Single adapter (max_loras == 1): the per-adapter virtual-expert offset - # (safe_lora * num_experts) is always 0, so align never emits an id >= - # virtual_num_experts and fused_sanitize_expert_ids is an identity -> skip it. - if max_loras != 1: - expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts) result = ( sorted_token_ids, expert_ids, @@ -673,11 +735,42 @@ def _merged_experts_fused_moe_lora_add_impl( assert intermediate_buffer is not None assert intermediate_buffer.ndim == 2 assert intermediate_buffer.shape[0] == token_lora_mapping.shape[0] - lora_a_virtual = _merge_lora_expert_weight(lora_a) - lora_b_virtual = _merge_lora_expert_weight(lora_b) + if num_routes == 0: + if stage == "shrink": + if intermediate_buffer is not None: + return intermediate_buffer + return hidden_states.new_empty( + (token_lora_mapping.shape[0], topk_ids.shape[1], max_lora_rank) + ) + if stage == "expand": + assert intermediate_buffer is not None, ( + "stage='expand' requires intermediate_buffer" + ) + return None + num_experts_a = lora_a.shape[1] num_experts_b = lora_b.shape[1] - b_stage_config = _get_stage_config(lora_b_virtual, 1) + a_shape = (max_loras * num_experts_a, lora_a.shape[2], lora_a.shape[3]) + b_shape = (lora_b.shape[0] * num_experts_b, lora_b.shape[2], lora_b.shape[3]) + tokens = token_lora_mapping.shape[0] + needs_shrink_config = stage in ("all", "shrink") or ( + stage == "routing" and prewarm_a_routing + ) + reuse_b_block = ( + needs_shrink_config + and tokens >= 512 + and lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get() + ) + needs_b_config = ( + stage in ("all", "expand") + or (prewarm_b_routing and (stage == "routing" or routing_cache is not None)) + or reuse_b_block + ) + b_stage_config = ( + _get_stage_config(b_shape, 1, hidden_states.dtype, tokens) + if needs_b_config + else None + ) if stage == "routing": # Pre-warm the routing cache on the CALLER'S (main) stream so the @@ -687,20 +780,12 @@ def _merged_experts_fused_moe_lora_add_impl( # allocator's stream tracking is disabled while capturing), corrupting # replays. Routing needs only topk_ids + token_lora_mapping, which are # both ready before the side-stream fork, so it can run on main. - a_cfg = _get_shrink_stage_config(lora_a_virtual, token_lora_mapping.shape[0]) - if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get(): - a_cfg = {**a_cfg, "BLOCK_SIZE_M": 16} - # Match the actual shrink-stage override below. Without this, callers - # that admit prefill-shaped batches into a side stream prewarm block-32 - # routing here, then miss the cache when shrink switches to the B-stage - # block size. The miss allocates routing buffers on the side stream - # during capture, violating the allocation guarantee of stage='routing'. - if ( - lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get() - and token_lora_mapping.shape[0] >= 512 - ): - a_cfg["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"] if prewarm_a_routing: + a_cfg = _get_shrink_stage_config( + a_shape, + tokens, + b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None, + ) _get_routing( topk_ids, token_lora_mapping, @@ -709,6 +794,7 @@ def _merged_experts_fused_moe_lora_add_impl( a_cfg["BLOCK_SIZE_M"], ) if prewarm_b_routing: + assert b_stage_config is not None _get_routing( topk_ids, token_lora_mapping, @@ -721,22 +807,9 @@ def _merged_experts_fused_moe_lora_add_impl( intermediate = intermediate_buffer if stage != "expand": a_stage_config = _get_shrink_stage_config( - lora_a_virtual, token_lora_mapping.shape[0] + a_shape, tokens, b_stage_config["BLOCK_SIZE_M"] if reuse_b_block else None ) - if lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE.get(): - # Test-only override; the launcher fixes N and K block sizes. - a_stage_config = { - **a_stage_config, - "BLOCK_SIZE_M": 16, - "num_warps": 4, - "num_stages": 4, - } - # Match routing block sizes so prefill stages can share cached alignment. - if ( - lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE.get() - and token_lora_mapping.shape[0] >= 512 - ): - a_stage_config["BLOCK_SIZE_M"] = b_stage_config["BLOCK_SIZE_M"] + lora_a_virtual = _merge_lora_expert_weight(lora_a) ( sorted_token_ids, expert_ids, @@ -800,6 +873,7 @@ def _merged_experts_fused_moe_lora_add_impl( # Pre-warm the routing-B cache on this (side) stream so the later "expand" stage # launches no routing kernels — they overlap finalize together with the shrink. if routing_cache is not None and prewarm_b_routing: + assert b_stage_config is not None _get_routing( topk_ids, token_lora_mapping, @@ -810,6 +884,8 @@ def _merged_experts_fused_moe_lora_add_impl( return intermediate assert intermediate is not None, "stage='expand' requires intermediate_buffer" + assert b_stage_config is not None + lora_b_virtual = _merge_lora_expert_weight(lora_b) ( sorted_token_ids, expert_ids, @@ -946,6 +1022,8 @@ def merged_experts_fused_moe_lora_add( prewarm_a_routing: bool = True, prewarm_b_routing: bool = True, zero_intermediate: bool = False, + include_inactive_routes: bool = False, + adapter_enabled: torch.Tensor | None = None, ) -> "torch.Tensor | None": """Public API: wraps the registered op with routing_cache support.""" return _merged_experts_fused_moe_lora_add_impl( @@ -972,4 +1050,6 @@ def merged_experts_fused_moe_lora_add( prewarm_a_routing=prewarm_a_routing, prewarm_b_routing=prewarm_b_routing, zero_intermediate=zero_intermediate, + include_inactive_routes=include_inactive_routes, + adapter_enabled=adapter_enabled, ) diff --git a/python/sglang/kernels/ops/lora/moe/virtual_experts.py b/python/sglang/kernels/ops/lora/moe/virtual_experts.py index cd194e77b7..6d2f702a52 100644 --- a/python/sglang/kernels/ops/lora/moe/virtual_experts.py +++ b/python/sglang/kernels/ops/lora/moe/virtual_experts.py @@ -20,22 +20,21 @@ def _fused_virtual_topk_ids_kernel( token_lora_mapping_ptr, virtual_topk_ids_ptr, token_lora_mask_ptr, + adapter_enabled_ptr, + stride_enabled, num_experts_for_weight: tl.constexpr, + SHARED_OUTER: tl.constexpr, + NUM_SLOTS: tl.constexpr, + HAS_ADAPTER_ENABLED: tl.constexpr, M, top_k: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """ - Fuses _get_virtual_topk_ids: comparison + clamp + arithmetic into one kernel. + Map active slots and valid base experts to virtual weight indices. - For each (m, k): - lora_id = token_lora_mapping[m] - mask[m] = (lora_id >= 0) - safe_lora = max(lora_id, 0) - if shared_outer: (handled by num_experts_for_weight == 0 sentinel) - virtual_topk_ids[m, k] = safe_lora * 1 (= safe_lora) - else: - virtual_topk_ids[m, k] = topk_ids[m, k] + safe_lora * num_experts_for_weight + Invalid slots or base experts map to -1. Shared weights accept any + nonnegative base expert and use the adapter slot as the weight index. """ pid = tl.program_id(0) offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) @@ -46,16 +45,29 @@ def _fused_virtual_topk_ids_kernel( # k = offs % top_k # not needed directly lora_id = tl.load(token_lora_mapping_ptr + m, mask=valid, other=0) - mask_val = lora_id >= 0 - safe_lora = tl.maximum(lora_id, 0) + mask_val = (lora_id >= 0) & (lora_id < NUM_SLOTS) + if HAS_ADAPTER_ENABLED: + if NUM_SLOTS == 1: + mask_val = mask_val & (tl.load(adapter_enabled_ptr) != 0) + else: + mask_val = mask_val & ( + tl.load( + adapter_enabled_ptr + lora_id * stride_enabled, + mask=valid & mask_val, + other=0, + ) + != 0 + ) + safe_lora = tl.where(mask_val, lora_id, 0) base = tl.load(topk_ids_ptr + offs, mask=valid, other=0) - # Preserve negative sentinel topk_ids (e.g. -1 for non-local experts after - # EP dispatch). Without this, `-1 + safe_lora * num_experts` would land on - # a real virtual-expert slot belonging to another adapter and trigger OOB - # loads in downstream LoRA kernels. - shifted = base + safe_lora * num_experts_for_weight - result = tl.where(base < 0, base, shifted) + valid_base = base >= 0 + if not SHARED_OUTER: + valid_base = valid_base & (base < num_experts_for_weight) + expert = 0 if SHARED_OUTER else tl.where(valid_base, base, 0) + shifted = expert + safe_lora * num_experts_for_weight + # All align providers use one sentinel bucket; borrowed route IDs stay intact. + result = tl.where(mask_val & valid_base, shifted, -1) tl.store(virtual_topk_ids_ptr + offs, result, mask=valid) # Write mask once per row (at first k position) @@ -70,6 +82,7 @@ def _fused_virtual_topk_ids( num_experts: int, shared_outer: bool, max_loras: int, + adapter_enabled: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, int]: """ Returns virtual topk_ids, token_lora_mask, and virtual_num_experts. @@ -77,27 +90,27 @@ def _fused_virtual_topk_ids( M, top_k = topk_ids.shape device = topk_ids.device - if shared_outer: - num_experts_for_weight = 1 - # For shared_outer, we need topk_ids to be zeros - zero_topk = torch.zeros_like(topk_ids) - input_topk = zero_topk - else: - num_experts_for_weight = num_experts - input_topk = topk_ids + num_experts_for_weight = 1 if shared_outer else num_experts virtual_topk_ids = torch.empty_like(topk_ids) token_lora_mask = torch.empty(M, dtype=torch.bool, device=device) + if M * top_k == 0: + return virtual_topk_ids, token_lora_mask, num_experts_for_weight * max_loras BLOCK_SIZE = 1024 grid = ((M * top_k + BLOCK_SIZE - 1) // BLOCK_SIZE,) _fused_virtual_topk_ids_kernel[grid]( - input_topk, + topk_ids, token_lora_mapping, virtual_topk_ids, token_lora_mask, + adapter_enabled, + adapter_enabled.stride(0) if adapter_enabled is not None else 0, num_experts_for_weight, + shared_outer, + max_loras, + adapter_enabled is not None, M, top_k, BLOCK_SIZE, @@ -107,48 +120,6 @@ def _fused_virtual_topk_ids( return virtual_topk_ids, token_lora_mask, virtual_num_experts -@triton.jit -def _fused_sanitize_expert_ids_kernel( - expert_ids_ptr, - output_ptr, - num_virtual_experts, - N, - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(0) - offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - valid = offs < N - - eid = tl.load(expert_ids_ptr + offs, mask=valid, other=0) - result = tl.where(eid < num_virtual_experts, eid, -1) - tl.store(output_ptr + offs, result, mask=valid) - - -def fused_sanitize_expert_ids( - expert_ids: torch.Tensor, - num_virtual_experts: int, -) -> torch.Tensor: - """ - Sanitize expert_ids by replacing values >= num_virtual_experts with -1. - - Returns a new tensor with expert_ids >= num_virtual_experts replaced by -1. - """ - N = expert_ids.numel() - output = torch.empty_like(expert_ids) - - BLOCK_SIZE = 1024 - grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,) - - _fused_sanitize_expert_ids_kernel[grid]( - expert_ids, - output, - num_virtual_experts, - N, - BLOCK_SIZE, - ) - return output - - @triton.jit def _moe_lora_shrink_splitk_kernel( # Pointers @@ -187,6 +158,9 @@ def _moe_lora_shrink_splitk_kernel( num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M) num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + if pid_mn >= num_pid_m * num_pid_n: + return + num_pid_in_group = GROUP_SIZE_M * num_pid_n group_id = pid_mn // num_pid_in_group first_pid_m = group_id * GROUP_SIZE_M @@ -194,9 +168,6 @@ def _moe_lora_shrink_splitk_kernel( pid_m = first_pid_m + ((pid_mn % num_pid_in_group) % group_size_m) pid_n = (pid_mn % num_pid_in_group) // group_size_m - if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: - return - # Token routing (same pattern as fused_moe_triton_kernels) offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64) @@ -309,7 +280,7 @@ def _align_block_size_jit( scratch=None, *, return_inactive_prefix_count: bool = False, - map_invalid_experts_to_sentinel: bool = False, + map_invalid_experts_to_sentinel: bool = True, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """CUDA JIT align_block_size for num_experts > 1024 (up to 8191). @@ -319,8 +290,8 @@ def _align_block_size_jit( ``scratch(numel)`` returns the int32 scratch buffer (default: a fresh allocation); a caller under CUDA graphs passes a persistent buffer. - IDs must be certified in [-1, num_experts) unless - ``map_invalid_experts_to_sentinel`` maps other signed IDs to bucket zero. + Out-of-range signed IDs share the sentinel bucket; opt out only for IDs + certified in [-1, num_experts). """ assert num_experts <= 8191, ( f"_align_block_size_jit supports at most 8191 experts " @@ -544,6 +515,9 @@ def _merged_experts_fused_moe_lora_add_impl( experts_shared_outer_loras_a: bool, experts_shared_outer_loras_b: bool, routing_cache: dict | None = None, + adapter_enabled: torch.Tensor | None = None, + *, + a_output_slices: int = 1, ) -> None: """ 1. Prepare virtual expert routing metadata from topk_ids + token_lora_mapping * num_experts. @@ -551,8 +525,21 @@ def _merged_experts_fused_moe_lora_add_impl( 3. Run regular SGLang fused-MoE kernels for LoRA A and LoRA B. 4. Mask out tokens with token_lora_mapping == -1 on the add path. """ + assert a_output_slices in (1, 2) + if a_output_slices == 2: + assert lora_a.shape[2] == 2 * lora_b.shape[3] + assert lora_b.shape[2] > 0 and lora_b.shape[2] % 2 == 0 + assert output.shape[-1] == lora_b.shape[2] max_loras, _, max_lora_rank, _ = lora_a.shape - input_top_k = 1 if hidden_states.shape[0] == topk_ids.numel() else topk_ids.shape[1] + num_routes = topk_ids.numel() + input_top_k = 1 if hidden_states.shape[0] == num_routes else topk_ids.shape[1] + if adapter_enabled is not None: + assert adapter_enabled.shape == (max_loras,) + assert adapter_enabled.device == topk_ids.device + assert adapter_enabled.dtype == torch.int32 + + if num_routes == 0: + return def _merge_lora_expert_weight(t: torch.Tensor) -> torch.Tensor: # [max_loras, num_experts, x, y] -> [max_loras * num_experts, x, y] @@ -619,7 +606,7 @@ def _merged_experts_fused_moe_lora_add_impl( block_size: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: # Check routing_cache for cross-call reuse (gate_up and down share routing) - cache_key = (num_experts, shared_outer, block_size) + cache_key = (num_experts, shared_outer, block_size, id(adapter_enabled)) if routing_cache is not None: cached = routing_cache.get(cache_key) if cached is not None: @@ -627,7 +614,12 @@ def _merged_experts_fused_moe_lora_add_impl( virtual_topk_ids, token_lora_mask, virtual_num_experts = ( _fused_virtual_topk_ids( - topk_ids, token_lora_mapping, num_experts, shared_outer, max_loras + topk_ids, + token_lora_mapping, + num_experts, + shared_outer, + max_loras, + adapter_enabled=adapter_enabled, ) ) sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size( @@ -645,7 +637,6 @@ def _merged_experts_fused_moe_lora_add_impl( ) sorted_token_ids = sorted_token_ids[:tight_padded] expert_ids = expert_ids[: tight_padded // block_size] - expert_ids = fused_sanitize_expert_ids(expert_ids, virtual_num_experts) result = ( sorted_token_ids, expert_ids, @@ -739,6 +730,7 @@ def _merged_experts_fused_moe_lora_add_impl( fuse_add_to_output=True, add_output_mask=token_lora_mask, router_topk=topk_ids.shape[1], + a_output_slices=a_output_slices, ) @@ -790,8 +782,11 @@ def merged_experts_fused_moe_lora_add( experts_shared_outer_loras_a: bool, experts_shared_outer_loras_b: bool, routing_cache: dict | None = None, + adapter_enabled: torch.Tensor | None = None, + *, + a_output_slices: int = 1, ) -> None: - """Public API: wraps the registered op with routing_cache support.""" + """Add virtual-expert LoRA deltas, optionally pairing A rank/output halves.""" _merged_experts_fused_moe_lora_add_impl( output, hidden_states, @@ -804,4 +799,6 @@ def merged_experts_fused_moe_lora_add( experts_shared_outer_loras_a, experts_shared_outer_loras_b, routing_cache, + adapter_enabled=adapter_enabled, + a_output_slices=a_output_slices, ) diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index 2b95264024..babf74af8e 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -382,6 +382,9 @@ def fused_moe_kernel( FUSE_SWIGLU: tl.constexpr = False, USE_GDC: tl.constexpr = False, GDC_EARLY: tl.constexpr = False, + A_OUTPUT_SLICES: tl.constexpr = 1, + A_OUTPUT_SLICE_WIDTH: tl.constexpr = 0, + A_OUTPUT_SLICE_K: tl.constexpr = 0, ): """ Implements the fused computation for a Mixture of Experts (MOE) using @@ -420,7 +423,14 @@ def fused_moe_kernel( # This is done in a grouped ordering to promote L2 data reuse. pid = tl.program_id(axis=0) num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + if A_OUTPUT_SLICES == 2: + # Gate/up outputs use separate rank halves of the same shrink result. + # Keep each N tile inside one half, including ragged output widths. + slice_width: tl.constexpr = A_OUTPUT_SLICE_WIDTH + num_pid_n_per_slice: tl.constexpr = tl.cdiv(slice_width, BLOCK_SIZE_N) + num_pid_n = 2 * num_pid_n_per_slice + else: + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) num_pid_in_group = GROUP_SIZE_M * num_pid_n group_id = pid // num_pid_in_group first_pid_m = group_id * GROUP_SIZE_M @@ -445,7 +455,11 @@ def fused_moe_kernel( off_experts_i32 = tl.load(expert_ids_ptr + pid_m) off_experts = off_experts_i32.to(tl.int64) - if off_experts == -1: + if A_OUTPUT_SLICES == 2: + inactive_expert = off_experts < 0 + else: + inactive_expert = off_experts == -1 + if inactive_expert: # The down GEMM never reads inactive sorted or half-width intermediates. # Add/reduce outputs already hold their initial value and must stay intact. if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE): @@ -463,7 +477,15 @@ def fused_moe_kernel( ) return - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N + if A_OUTPUT_SLICES == 2: + output_slice = pid_n // num_pid_n_per_slice + local_pid_n = pid_n % num_pid_n_per_slice + local_columns = local_pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to( + tl.int64 + ) + offs_bn = local_columns % slice_width + output_slice * slice_width + else: + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N offs_k = tl.arange(0, BLOCK_SIZE_K) if a_desc is not None: assert use_fp8_w8a8 and group_n > 0 and group_k > 0 @@ -472,6 +494,8 @@ def fused_moe_kernel( a_ptrs = a_ptr + ( offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak ) + if A_OUTPUT_SLICES == 2: + a_ptrs += output_slice * A_OUTPUT_SLICE_K * stride_ak if b_desc is not None: start_offs_n = pid_n * BLOCK_SIZE_N @@ -610,7 +634,12 @@ def fused_moe_kernel( accumulator = accumulator.to(compute_type) # ----------------------------------------------------------- # Write back the block of the output - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + if A_OUTPUT_SLICES == 2: + offs_cn = local_columns + output_slice * slice_width + output_column_mask = local_columns < slice_width + else: + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + output_column_mask = offs_cn < N if FUSE_SWIGLU: # W13 rows were interleaved at load time, so gate/up of the same @@ -667,7 +696,7 @@ def fused_moe_kernel( offs_token_out = offs_token // ROUTER_TOPK add_mask = tl.load(add_mask_ptr + offs_token_out, mask=token_mask, other=False) c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] - c_mask = token_mask[:, None] & add_mask[:, None] & (offs_cn[None, :] < N) + c_mask = token_mask[:, None] & add_mask[:, None] & output_column_mask[None, :] existing = tl.load(c_ptrs, mask=c_mask, other=0.0) tl.store(c_ptrs, existing + accumulator, mask=c_mask) # ===== TO BE REFACTORED ==== @@ -810,10 +839,38 @@ def invoke_fused_moe_kernel( mask_output: bool = False, lora_preserve_base: bool = False, fuse_swiglu: bool = False, + a_output_slices: int = 1, ) -> None: - """Inactive expert IDs are always skipped, including with filter_expert=False.""" + """Skip inactive experts, including with filter_expert=False. + + a_output_slices=2 pairs consecutive A rank halves with B/C output halves + on the unquantized add path, without splitting the shrink/expand launches. + """ assert topk_weights.stride(1) == 1 assert sorted_token_ids.stride(0) == 1 + assert a_output_slices in (1, 2) + a_output_slice_width = 0 + a_output_slice_k = 0 + if a_output_slices == 2: + assert fuse_add_to_output + assert add_output_mask is not None + assert not mul_routed_weight + assert not (use_fp8_w8a8 or use_int8_w8a8 or use_int8_w8a16 or use_int4_w4a16) + assert not per_channel_quant and block_shape is None + assert not (a_use_tma or b_use_tma or fuse_swiglu or c_sorted) + assert not ( + fuse_sum_all_reduce or mask_output or lora_preserve_base or no_combine + ) + assert bias is None and A_scale is None and B_scale is None and B_zp is None + assert A.dtype in (torch.float16, torch.bfloat16) + assert B.dtype == C.dtype == A.dtype + assert compute_type == (tl.float16 if A.dtype == torch.float16 else tl.bfloat16) + assert A.ndim == 2 and B.ndim == 3 + assert A.shape[1] == 2 * B.shape[2] + assert B.shape[1] > 0 and B.shape[1] % 2 == 0 + assert C.shape[-1] == B.shape[1] + a_output_slice_width = B.shape[1] // 2 + a_output_slice_k = B.shape[2] if fuse_swiglu: # The epilogue assumes an interleaved-gate/up bf16 up-GEMM writing a @@ -887,7 +944,11 @@ def invoke_fused_moe_kernel( grid = lambda META: ( triton.cdiv(sorted_token_ids.shape[0], META["BLOCK_SIZE_M"]) - * triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]), + * ( + triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]) + if a_output_slices == 1 + else 2 * triton.cdiv(a_output_slice_width, META["BLOCK_SIZE_N"]) + ), ) K = B.shape[2] - padded_size @@ -1037,6 +1098,9 @@ def invoke_fused_moe_kernel( c_sorted=c_sorted, swap_ab=swap_ab, FUSE_ADD_TO_OUTPUT=fuse_add_to_output, + A_OUTPUT_SLICES=a_output_slices, + A_OUTPUT_SLICE_WIDTH=a_output_slice_width, + A_OUTPUT_SLICE_K=a_output_slice_k, MASK_OUTPUT=mask_output, FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce, ROUTER_TOPK=router_topk, diff --git a/python/sglang/srt/lora/lora_moe_runner_marlin.py b/python/sglang/srt/lora/lora_moe_runner_marlin.py index a7a42821b9..92e6384ac6 100644 --- a/python/sglang/srt/lora/lora_moe_runner_marlin.py +++ b/python/sglang/srt/lora/lora_moe_runner_marlin.py @@ -72,7 +72,7 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output topk_ids = topk_output.topk_ids - topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0) + topk_weights = topk_output.topk_weights assert runner_config.activation == "silu", "Only SiLU activation is supported." assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, ( @@ -96,7 +96,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): quant_info.global_num_experts if quant_info.expert_map is not None else E ) sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - topk_ids, block_size_m, align_num_experts + topk_ids, + block_size_m, + align_num_experts, + return_inactive_prefix_count=quant_info.expert_map is None, ) # Per-call workspace like fused_experts_none_to_marlin: a shared buffer aliases @@ -119,13 +122,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): ) # Stage 1: Gate/Up (Marlin) - projection_cache = torch.zeros( - (M * topk * (2 * N + K),), - device=hidden_states.device, - dtype=hidden_states.dtype, + intermediate_cache1 = torch.empty( + (M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype ) - intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N) - intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K) intermediate_cache1 = moe_wna16_marlin_gemm( hidden_states, intermediate_cache1, @@ -145,6 +144,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): top_k=topk, mul_topk_weights=False, is_ep=quant_info.expert_map is not None, + expert_ids_are_sorted=quant_info.expert_map is None, + has_inactive_prefix_count=quant_info.expert_map is None, b_q_type=scalar_type1, size_m=M, size_n=2 * N, @@ -169,6 +170,9 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2) # Stage 3: Down (Marlin) + intermediate_cache3 = torch.empty( + (M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype + ) intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, @@ -188,7 +192,10 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): moe_block_size=block_size_m, top_k=1, mul_topk_weights=True, + zero_inactive_output=True, is_ep=quant_info.expert_map is not None, + expert_ids_are_sorted=quant_info.expert_map is None, + has_inactive_prefix_count=quant_info.expert_map is None, b_q_type=scalar_type2, size_m=M * topk, size_n=K, diff --git a/python/sglang/srt/lora/lora_moe_runners.py b/python/sglang/srt/lora/lora_moe_runners.py index adddaf10d2..e65ba646c2 100644 --- a/python/sglang/srt/lora/lora_moe_runners.py +++ b/python/sglang/srt/lora/lora_moe_runners.py @@ -189,7 +189,12 @@ class LoRAInfo: @dataclass class LoRAHooks: - """Hook callbacks for injecting LoRA deltas into the MoE pipeline.""" + """Hook callbacks for injecting LoRA deltas into the MoE pipeline. + + Dropped route IDs remain negative. Hooks must skip those routes before + reading their payloads or scores; private inactive intermediates may be + uninitialized and routing scores are passed without a sanitized copy. + """ after_gate_up: ( Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], None] | None @@ -357,6 +362,8 @@ def _add_lora_gate_up_delta( experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras, experts_shared_outer_loras_b=False, routing_cache=routing_cache, + adapter_enabled=lora_info.adapter_enabled, + a_output_slices=2 if is_gated else 1, ) else: blk = _get_moe_lora_block_config(r) @@ -432,6 +439,7 @@ def _add_lora_down_delta( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras, routing_cache=routing_cache, + adapter_enabled=lora_info.adapter_enabled, ) else: blk = _get_moe_lora_block_config(lora_info.max_lora_rank) diff --git a/python/sglang/srt/lora/marlin_lora_temp/activation.py b/python/sglang/srt/lora/marlin_lora_temp/activation.py index 49d7c8c150..d31cadc875 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/activation.py +++ b/python/sglang/srt/lora/marlin_lora_temp/activation.py @@ -14,11 +14,34 @@ import triton import triton.language as tl +@triton.jit +def _silu_mul_add_delta_row( + x_ptr, + d_ptr, + out_ptr, + N, + row, + cols, + mask, + delta_mask, +): + x_row = x_ptr + row * 2 * N + d_row = d_ptr + row * 2 * N + gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32) + up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32) + gate += tl.load(d_row + cols, mask=delta_mask, other=0.0).to(tl.float32) + up += tl.load(d_row + N + cols, mask=delta_mask, other=0.0).to(tl.float32) + + silu = gate * tl.sigmoid(gate) + out = silu * up + tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask) + + @triton.jit def _silu_mul_add_delta_kernel( - x_ptr, # [T, 2N] gemm1 output, [gate || up] - d_ptr, # [T, 2N] LoRA gate_up delta, same layout - out_ptr, # [T, N] + x_ptr, + d_ptr, + out_ptr, N, BLOCK: tl.constexpr, ): @@ -26,17 +49,7 @@ def _silu_mul_add_delta_kernel( block = tl.program_id(1) cols = block * BLOCK + tl.arange(0, BLOCK) mask = cols < N - - x_row = x_ptr + row * 2 * N - d_row = d_ptr + row * 2 * N - gate = tl.load(x_row + cols, mask=mask, other=0.0).to(tl.float32) - up = tl.load(x_row + N + cols, mask=mask, other=0.0).to(tl.float32) - gate += tl.load(d_row + cols, mask=mask, other=0.0).to(tl.float32) - up += tl.load(d_row + N + cols, mask=mask, other=0.0).to(tl.float32) - - silu = gate * tl.sigmoid(gate) - out = silu * up - tl.store(out_ptr + row * N + cols, out.to(out_ptr.dtype.element_ty), mask=mask) + _silu_mul_add_delta_row(x_ptr, d_ptr, out_ptr, N, row, cols, mask, mask) def silu_and_mul_add_delta( @@ -44,7 +57,7 @@ def silu_and_mul_add_delta( gate_up_delta: torch.Tensor, out: torch.Tensor, ) -> None: - """``out = silu(g + dg) * (u + du)`` over contiguous ``[T, 2N]`` inputs.""" + """Both producers must define inactive route rows and disabled deltas as zero.""" T, two_n = gemm1_out.shape N = two_n // 2 assert gate_up_delta.shape == (T, two_n) and out.shape == (T, N) diff --git a/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py b/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py index 736e72409a..689adabd7b 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py +++ b/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py @@ -24,6 +24,7 @@ def _direct_gate_expand_kernel( gate_b_ptr, # [S, E, 2I, R] topk_ids_ptr, # [M, topk] token_lora_mapping_ptr, # [M] + adapter_enabled_ptr, output_ptr, # [M, topk, 2I] stride_as, stride_am, @@ -34,11 +35,13 @@ def _direct_gate_expand_kernel( stride_br, stride_oq, stride_on, + stride_enabled, ROUTER_TOPK: tl.constexpr, INTERMEDIATE_SIZE: tl.constexpr, RANK: tl.constexpr, GATE_WIDTH: tl.constexpr, NUM_SLOTS: tl.constexpr, + HAS_ADAPTER_ENABLED: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, ): @@ -48,6 +51,15 @@ def _direct_gate_expand_kernel( token = route // ROUTER_TOPK slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64) active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0) + if HAS_ADAPTER_ENABLED: + active = active & ( + tl.load( + adapter_enabled_ptr + slot * stride_enabled, + mask=(slot >= 0) & (slot < NUM_SLOTS), + other=0, + ) + != 0 + ) expert = tl.maximum(expert, 0) offs_m = tl.arange(0, BLOCK_M) offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) @@ -91,6 +103,7 @@ def _direct_down_shrink_kernel( down_a_ptr, # [S, E, R, I] topk_ids_ptr, # [M, topk] token_lora_mapping_ptr, # [M] + adapter_enabled_ptr, output_ptr, # [M, topk, R] stride_xq, stride_xk, @@ -100,10 +113,12 @@ def _direct_down_shrink_kernel( stride_ak, stride_oq, stride_or, + stride_enabled, ROUTER_TOPK: tl.constexpr, INTERMEDIATE_SIZE: tl.constexpr, RANK: tl.constexpr, NUM_SLOTS: tl.constexpr, + HAS_ADAPTER_ENABLED: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr, ): @@ -112,6 +127,15 @@ def _direct_down_shrink_kernel( token = route // ROUTER_TOPK slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64) active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0) + if HAS_ADAPTER_ENABLED: + active = active & ( + tl.load( + adapter_enabled_ptr + slot * stride_enabled, + mask=(slot >= 0) & (slot < NUM_SLOTS), + other=0, + ) + != 0 + ) expert = tl.maximum(expert, 0) offs_m = tl.arange(0, BLOCK_M) offs_r = tl.arange(0, RANK) @@ -153,6 +177,8 @@ def direct_decode_gate_expand( topk_ids: torch.Tensor, token_lora_mapping: torch.Tensor, output: torch.Tensor, + *, + adapter_enabled: torch.Tensor | None = None, ) -> None: """Expand shared gate/up rank vectors through directly selected experts. @@ -163,6 +189,10 @@ def direct_decode_gate_expand( """ num_tokens = topk_ids.shape[0] + if adapter_enabled is not None: + assert adapter_enabled.shape == (gate_b.shape[0],) + assert adapter_enabled.device == topk_ids.device + assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64) if shared_intermediate.ndim == 2: shared_slot_stride = 0 shared_token_stride = shared_intermediate.stride(0) @@ -180,6 +210,7 @@ def direct_decode_gate_expand( gate_b, topk_ids, token_lora_mapping, + adapter_enabled, output_flat, shared_slot_stride, shared_token_stride, @@ -190,11 +221,13 @@ def direct_decode_gate_expand( gate_b.stride(3), output_flat.stride(0), output_flat.stride(1), + adapter_enabled.stride(0) if adapter_enabled is not None else 0, ROUTER_TOPK=ROUTER_TOPK, INTERMEDIATE_SIZE=intermediate_size, RANK=RANK, GATE_WIDTH=gate_width, NUM_SLOTS=gate_b.shape[0], + HAS_ADAPTER_ENABLED=adapter_enabled is not None, BLOCK_M=_BLOCK_M, BLOCK_N=_GATE_BLOCK_N, num_warps=4, @@ -208,6 +241,8 @@ def direct_decode_down_shrink( topk_ids: torch.Tensor, token_lora_mapping: torch.Tensor, output: torch.Tensor, + *, + adapter_enabled: torch.Tensor | None = None, ) -> None: """Shrink directly routed activations through per-expert LoRA-A. @@ -216,6 +251,10 @@ def direct_decode_down_shrink( """ num_tokens = topk_ids.shape[0] + if adapter_enabled is not None: + assert adapter_enabled.shape == (down_a.shape[0],) + assert adapter_enabled.device == topk_ids.device + assert adapter_enabled.dtype in (torch.bool, torch.int32, torch.int64) intermediate_size = activation.shape[1] output_flat = output.view(num_tokens * ROUTER_TOPK, RANK) num_routes = topk_ids.numel() @@ -224,6 +263,7 @@ def direct_decode_down_shrink( down_a, topk_ids, token_lora_mapping, + adapter_enabled, output_flat, activation.stride(0), activation.stride(1), @@ -233,10 +273,12 @@ def direct_decode_down_shrink( down_a.stride(3), output_flat.stride(0), output_flat.stride(1), + adapter_enabled.stride(0) if adapter_enabled is not None else 0, ROUTER_TOPK=ROUTER_TOPK, INTERMEDIATE_SIZE=intermediate_size, RANK=RANK, NUM_SLOTS=down_a.shape[0], + HAS_ADAPTER_ENABLED=adapter_enabled is not None, BLOCK_M=_BLOCK_M, BLOCK_K=_DOWN_BLOCK_K, num_warps=4, diff --git a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py index 0d89f3a276..4c3f9fd644 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py +++ b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py @@ -209,6 +209,7 @@ def fused_experts_experimental_sgl_marlin_lora( lora_info, ) -> StandardCombineInput: from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( + _prewarm_per_expert_lora_routes, merged_experts_fused_moe_lora_add, ) from sglang.srt.layers.moe.token_dispatcher.standard import ( @@ -233,7 +234,7 @@ def fused_experts_experimental_sgl_marlin_lora( hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output topk_ids = topk_output.topk_ids - topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0) + topk_weights = topk_output.topk_weights if ( topk_ids.ndim != 2 or topk_weights.shape != topk_ids.shape @@ -345,8 +346,25 @@ def fused_experts_experimental_sgl_marlin_lora( != lora_info.gate_up_lora_b_weights.shape[-1] ) # Allocate and prewarm on main; side streams launch kernels only. - gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N)) - if not direct_decode: + gate_up_delta = hidden_states.new_empty((M, topk, 2 * N)) + paired_routing = ( + factored_shared_outer + and not direct_decode + and _prewarm_per_expert_lora_routes( + hidden_states, + lora_info.down_lora_a_weights, + lora_info.gate_up_lora_b_weights, + topk_ids, + lora_info.token_lora_mapping, + routing_cache, + shrink_lora_b=lora_info.down_lora_b_weights, + adapter_enabled=lora_info.adapter_enabled, + include_inactive_routes=True, + local_expert_offset=0, + local_num_experts=E, + ) + ) + if not direct_decode and not paired_routing: merged_experts_fused_moe_lora_add( output=gate_up_delta, hidden_states=hidden_states, @@ -359,7 +377,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras, experts_shared_outer_loras_b=False, routing_cache=routing_cache, + include_inactive_routes=True, stage="routing", + adapter_enabled=lora_info.adapter_enabled, prewarm_a_routing=not factored_shared_outer, local_expert_offset=0, local_num_experts=E, @@ -380,7 +400,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=True, experts_shared_outer_loras_b=False, routing_cache=collapsed_routing_cache, + include_inactive_routes=True, stage="routing", + adapter_enabled=lora_info.adapter_enabled, prewarm_a_routing=True, prewarm_b_routing=False, local_expert_offset=0, @@ -402,7 +424,7 @@ def fused_experts_experimental_sgl_marlin_lora( ) ) # Prewarm down routing and allocate its intermediate on main. - if not direct_decode: + if not direct_decode and not paired_routing: merged_experts_fused_moe_lora_add( output=gate_up_delta, hidden_states=hidden_states, @@ -415,8 +437,10 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras, routing_cache=routing_cache, + include_inactive_routes=True, stage="routing", prewarm_b_routing=not factored_shared_outer, + adapter_enabled=lora_info.adapter_enabled, local_expert_offset=0, local_num_experts=E, ) @@ -436,8 +460,10 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=True, routing_cache=collapsed_routing_cache, + include_inactive_routes=True, stage="routing", prewarm_a_routing=False, + adapter_enabled=lora_info.adapter_enabled, prewarm_b_routing=True, local_expert_offset=0, local_num_experts=E, @@ -476,7 +502,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=True, experts_shared_outer_loras_b=False, routing_cache=collapsed_routing_cache, + include_inactive_routes=True, stage="shrink", + adapter_enabled=lora_info.adapter_enabled, prewarm_b_routing=False, intermediate_buffer=gate_up_intermediate, local_expert_offset=0, @@ -495,6 +523,7 @@ def fused_experts_experimental_sgl_marlin_lora( topk_ids, lora_info.token_lora_mapping, gate_up_delta, + adapter_enabled=lora_info.adapter_enabled, ) return merged_experts_fused_moe_lora_add( @@ -509,7 +538,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=True, experts_shared_outer_loras_b=False, routing_cache=routing_cache, + include_inactive_routes=True, fuse_add_to_output=False, + adapter_enabled=lora_info.adapter_enabled, use_direct_expand_add=True, local_expert_offset=0, local_num_experts=E, @@ -530,7 +561,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=lora_info.experts_shared_outer_loras, experts_shared_outer_loras_b=False, routing_cache=routing_cache, + include_inactive_routes=True, fuse_add_to_output=False, + adapter_enabled=lora_info.adapter_enabled, use_direct_expand_add=use_direct_expand, local_expert_offset=0, local_num_experts=E, @@ -552,7 +585,11 @@ def fused_experts_experimental_sgl_marlin_lora( # Start alignment after the side-stream fork. EP IDs are already localized; # the wrapper owns Marlin's extra sentinel bucket. sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - topk_ids, block_size_m, E + topk_ids, + block_size_m, + E, + return_inactive_prefix_count=not ep_active, + map_invalid_experts_to_sentinel=False, ) # A cached workspace can outlive its CUDA graph-private pool. @@ -572,13 +609,9 @@ def fused_experts_experimental_sgl_marlin_lora( ) # Stage 1: gate_up (marlin) — concurrent with the side-stream delta above. - projection_cache = torch.zeros( - (M * topk * (2 * N + K),), - device=hidden_states.device, - dtype=hidden_states.dtype, + intermediate_cache1 = torch.empty( + (M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype ) - intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N) - intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K) intermediate_cache1 = moe_wna16_marlin_gemm( hidden_states, intermediate_cache1, @@ -597,7 +630,10 @@ def fused_experts_experimental_sgl_marlin_lora( moe_block_size=block_size_m, top_k=topk, mul_topk_weights=False, + zero_inactive_output=True, is_ep=ep_active, + expert_ids_are_sorted=not ep_active, + has_inactive_prefix_count=not ep_active, b_q_type=scalar_type1, size_m=M, size_n=2 * N, @@ -616,6 +652,8 @@ def fused_experts_experimental_sgl_marlin_lora( if run_lora: if lora_event is not None: torch.cuda.current_stream().wait_event(lora_event) + # Both private producers define dropped routes as zero; the runtime + # activation therefore needs no routing or adapter metadata reads. silu_and_mul_add_delta( intermediate_cache1.view(-1, 2 * N), gate_up_delta.view(-1, 2 * N), @@ -641,6 +679,7 @@ def fused_experts_experimental_sgl_marlin_lora( topk_ids, lora_info.token_lora_mapping, down_intermediate, + adapter_enabled=lora_info.adapter_enabled, ) else: merged_experts_fused_moe_lora_add( @@ -655,7 +694,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras, routing_cache=routing_cache, + include_inactive_routes=True, stage="shrink", + adapter_enabled=lora_info.adapter_enabled, prewarm_b_routing=not factored_shared_outer, intermediate_buffer=down_intermediate, local_expert_offset=0, @@ -673,6 +714,8 @@ def fused_experts_experimental_sgl_marlin_lora( down_rank_sum, routed_scaling_factor, block_m=_weighted_rank_sum_block_m(M), + topk_ids=topk_ids, + inactive_ranks_are_zero=True, ) down_shrink_done.record() if torch.cuda.is_current_stream_capturing(): @@ -682,6 +725,9 @@ def fused_experts_experimental_sgl_marlin_lora( silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2) # Stage 3: down (marlin). + intermediate_cache3 = torch.empty( + (M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype + ) intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, @@ -701,7 +747,10 @@ def fused_experts_experimental_sgl_marlin_lora( moe_block_size=block_size_m, top_k=1, mul_topk_weights=True, + zero_inactive_output=True, is_ep=ep_active, + expert_ids_are_sorted=not ep_active, + has_inactive_prefix_count=not ep_active, b_q_type=scalar_type2, size_m=M * topk, size_n=K, @@ -723,6 +772,7 @@ def fused_experts_experimental_sgl_marlin_lora( topk_ids, lora_info.token_lora_mapping, down_intermediate, + adapter_enabled=lora_info.adapter_enabled, ) else: down_intermediate.zero_() @@ -738,7 +788,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=True, routing_cache=routing_cache, + include_inactive_routes=True, stage="shrink", + adapter_enabled=lora_info.adapter_enabled, prewarm_b_routing=False, intermediate_buffer=down_intermediate, local_expert_offset=0, @@ -760,7 +812,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras, routing_cache=routing_cache, + include_inactive_routes=True, local_expert_offset=0, + adapter_enabled=lora_info.adapter_enabled, local_num_experts=E, zero_intermediate=lora_info.experts_shared_outer_loras, ) @@ -782,6 +836,7 @@ def fused_experts_experimental_sgl_marlin_lora( output, routed_scaling_factor, block_k=64, + topk_ids=topk_ids, ) else: tail_block_m, tail_block_k = fused_base_shared_lora_reduce_config(M) @@ -794,6 +849,7 @@ def fused_experts_experimental_sgl_marlin_lora( routed_scaling_factor, block_m=tail_block_m, block_k=tail_block_k, + topk_ids=topk_ids, ) elif routed_scaling_factor == 1.0 and M <= 512: torch.sum(intermediate_cache3, dim=1, out=output) @@ -810,6 +866,8 @@ def fused_experts_experimental_sgl_marlin_lora( down_rank_sum, routed_scaling_factor, block_m=_weighted_rank_sum_block_m(M), + topk_ids=topk_ids, + inactive_ranks_are_zero=True, ) else: # Wait only at the shared-B consumer. @@ -830,7 +888,9 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=True, routing_cache=collapsed_routing_cache, + include_inactive_routes=True, fuse_add_to_output=True, + adapter_enabled=lora_info.adapter_enabled, use_direct_expand_add=False, local_expert_offset=0, local_num_experts=E, @@ -854,8 +914,10 @@ def fused_experts_experimental_sgl_marlin_lora( experts_shared_outer_loras_a=False, experts_shared_outer_loras_b=lora_info.experts_shared_outer_loras, routing_cache=routing_cache, + include_inactive_routes=True, fuse_add_to_output=False, fuse_sum_all_reduce=True, + adapter_enabled=lora_info.adapter_enabled, use_direct_expand_add=lora_info.max_lora_rank <= 64, local_expert_offset=0, local_num_experts=E, diff --git a/python/sglang/srt/lora/marlin_lora_temp/shared_outer.py b/python/sglang/srt/lora/marlin_lora_temp/shared_outer.py index 5ea45ccd7b..cc3fae88d7 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/shared_outer.py +++ b/python/sglang/srt/lora/marlin_lora_temp/shared_outer.py @@ -16,6 +16,7 @@ import triton.language as tl def _weighted_topk_rank_sum_kernel( routed_rank_ptr, # [M, topk, R] topk_weights_ptr, # [M, topk] + topk_ids_ptr, output_ptr, # [M, R] M, R, @@ -25,9 +26,13 @@ def _weighted_topk_rank_sum_kernel( stride_rr, stride_wm, stride_wk, + stride_im, + stride_ik, stride_om, stride_or, TOPK: tl.constexpr, + HAS_TOPK_IDS: tl.constexpr, + INACTIVE_RANKS_ARE_ZERO: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_R: tl.constexpr, ): @@ -40,12 +45,23 @@ def _weighted_topk_rank_sum_kernel( accumulator = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32) for k in tl.static_range(TOPK): + route_mask = token_mask + if HAS_TOPK_IDS: + route = tl.load( + topk_ids_ptr + token * stride_im + k * stride_ik, + mask=token_mask, + other=-1, + ) + route_mask = route_mask & (route >= 0) + rank_load_mask = mask & route_mask[:, None] + if INACTIVE_RANKS_ARE_ZERO: + rank_load_mask = mask routed_rank = tl.load( routed_rank_ptr + token[:, None] * stride_rm + k * stride_rk + rank[None, :] * stride_rr, - mask=mask, + mask=rank_load_mask, other=0.0, ).to(tl.float32) weight = tl.load( @@ -53,6 +69,8 @@ def _weighted_topk_rank_sum_kernel( mask=token_mask, other=0.0, ).to(tl.float32) + if HAS_TOPK_IDS: + weight = tl.where(route_mask, weight, 0.0) accumulator += routed_rank * weight[:, None] accumulator *= scale @@ -69,18 +87,33 @@ def weighted_topk_rank_sum( output: torch.Tensor, routed_scaling_factor: float, *, + topk_ids: torch.Tensor | None = None, + inactive_ranks_are_zero: bool = False, block_m: int, ) -> None: - """Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``.""" + """Compute ``output[m] = scale * sum_k(gamma[m,k] * rank[m,k])``. + + ``inactive_ranks_are_zero`` certifies that the rank producer writes literal + zero for every dropped route. Original IDs select inactive scores to zero + before multiplication, excluding NaNs from those scores. The default also + masks rank loads and admits poisoned private payloads. + """ assert routed_rank.ndim == 3 assert topk_weights.shape == routed_rank.shape[:2] assert output.shape == (routed_rank.shape[0], routed_rank.shape[2]) assert routed_rank.is_contiguous() and topk_weights.is_contiguous() assert output.is_contiguous() + if inactive_ranks_are_zero: + assert topk_ids is not None if routed_rank.shape[0] == 0: return + if topk_ids is not None: + assert topk_ids.shape == topk_weights.shape + assert topk_ids.device == routed_rank.device + assert topk_ids.dtype in (torch.int32, torch.int64) + rank = routed_rank.shape[2] assert block_m in (1, 2) block_rank = max(16, triton.next_power_of_2(rank)) @@ -89,6 +122,7 @@ def weighted_topk_rank_sum( ]( routed_rank, topk_weights, + topk_ids, output, routed_rank.shape[0], rank, @@ -98,9 +132,13 @@ def weighted_topk_rank_sum( routed_rank.stride(2), topk_weights.stride(0), topk_weights.stride(1), + topk_ids.stride(0) if topk_ids is not None else 0, + topk_ids.stride(1) if topk_ids is not None else 0, output.stride(0), output.stride(1), TOPK=routed_rank.shape[1], + HAS_TOPK_IDS=topk_ids is not None, + INACTIVE_RANKS_ARE_ZERO=inactive_ranks_are_zero, BLOCK_M=block_m, BLOCK_R=block_rank, num_warps=1, @@ -112,6 +150,7 @@ def _fused_base_shared_lora_reduce_kernel( routed_base_ptr, # [M, topk, K], already weighted by the base MoE routed_rank_ptr, # [M, topk, R] topk_weights_ptr, # [M, topk] + topk_ids_ptr, shared_b_ptr, # [K, R] output_ptr, # [M, K] M, @@ -126,11 +165,14 @@ def _fused_base_shared_lora_reduce_kernel( stride_rr, stride_wm, stride_wk, + stride_im, + stride_ik, stride_sk, stride_sr, stride_om, stride_ok, TOPK: tl.constexpr, + HAS_TOPK_IDS: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr, BLOCK_R: tl.constexpr, @@ -147,12 +189,20 @@ def _fused_base_shared_lora_reduce_kernel( base_acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32) rank_acc = tl.zeros((BLOCK_M, BLOCK_R), dtype=tl.float32) for topk_idx in tl.static_range(TOPK): + route_mask = token_mask + if HAS_TOPK_IDS: + route = tl.load( + topk_ids_ptr + token * stride_im + topk_idx * stride_ik, + mask=token_mask, + other=-1, + ) + route_mask = route_mask & (route >= 0) base = tl.load( routed_base_ptr + token[:, None] * stride_bm + topk_idx * stride_bk + out_col[None, :] * stride_bn, - mask=token_mask[:, None] & out_mask[None, :], + mask=route_mask[:, None] & out_mask[None, :], other=0.0, ).to(tl.float32) routed_rank = tl.load( @@ -160,12 +210,12 @@ def _fused_base_shared_lora_reduce_kernel( + token[:, None] * stride_rm + topk_idx * stride_rk + rank[None, :] * stride_rr, - mask=token_mask[:, None] & rank_mask[None, :], + mask=route_mask[:, None] & rank_mask[None, :], other=0.0, ).to(tl.float32) weight = tl.load( topk_weights_ptr + token * stride_wm + topk_idx * stride_wk, - mask=token_mask, + mask=route_mask, other=0.0, ).to(tl.float32) base_acc += base @@ -200,6 +250,7 @@ def fused_base_shared_lora_reduce( output: torch.Tensor, routed_scaling_factor: float, *, + topk_ids: torch.Tensor | None = None, block_m: int, block_k: int, ) -> None: @@ -237,6 +288,11 @@ def fused_base_shared_lora_reduce( if routed_base.shape[0] == 0: return + if topk_ids is not None: + assert topk_ids.shape == topk_weights.shape + assert topk_ids.device == routed_rank.device + assert topk_ids.dtype in (torch.int32, torch.int64) + rank = routed_rank.shape[2] assert block_m in (1, 2, 4, 8) assert block_k in (32, 64, 128) @@ -250,6 +306,7 @@ def fused_base_shared_lora_reduce( routed_base, routed_rank, topk_weights, + topk_ids, shared_b, output, routed_base.shape[0], @@ -264,11 +321,14 @@ def fused_base_shared_lora_reduce( routed_rank.stride(2), topk_weights.stride(0), topk_weights.stride(1), + topk_ids.stride(0) if topk_ids is not None else 0, + topk_ids.stride(1) if topk_ids is not None else 0, shared_b.stride(0), shared_b.stride(1), output.stride(0), output.stride(1), TOPK=routed_base.shape[1], + HAS_TOPK_IDS=topk_ids is not None, BLOCK_M=block_m, BLOCK_K=block_k, BLOCK_R=block_rank, @@ -284,6 +344,7 @@ def _fused_base_mapped_shared_lora_reduce_kernel( routed_base_ptr, routed_rank_ptr, topk_weights_ptr, + topk_ids_ptr, shared_b_ptr, # [S, K, R] token_lora_mapping_ptr, output_ptr, @@ -298,12 +359,15 @@ def _fused_base_mapped_shared_lora_reduce_kernel( stride_rr, stride_wm, stride_wk, + stride_im, + stride_ik, stride_ss, stride_sk, stride_sr, stride_om, stride_ok, TOPK: tl.constexpr, + HAS_TOPK_IDS: tl.constexpr, NUM_SLOTS: tl.constexpr, BLOCK_K: tl.constexpr, BLOCK_R: tl.constexpr, @@ -317,12 +381,16 @@ def _fused_base_mapped_shared_lora_reduce_kernel( base_acc = tl.zeros((1, BLOCK_K), dtype=tl.float32) rank_acc = tl.zeros((1, BLOCK_R), dtype=tl.float32) for topk_idx in tl.static_range(TOPK): + route_active = tl.full((), True, tl.int1) + if HAS_TOPK_IDS: + route = tl.load(topk_ids_ptr + token * stride_im + topk_idx * stride_ik) + route_active = route >= 0 base = tl.load( routed_base_ptr + token * stride_bm + topk_idx * stride_bk + out_col[None, :] * stride_bn, - mask=out_mask[None, :], + mask=route_active & out_mask[None, :], other=0.0, ).to(tl.float32) routed_rank = tl.load( @@ -330,10 +398,14 @@ def _fused_base_mapped_shared_lora_reduce_kernel( + token * stride_rm + topk_idx * stride_rk + rank[None, :] * stride_rr, - mask=rank_mask[None, :], + mask=route_active & rank_mask[None, :], other=0.0, ).to(tl.float32) - weight = tl.load(topk_weights_ptr + token * stride_wm + topk_idx * stride_wk) + weight = tl.load( + topk_weights_ptr + token * stride_wm + topk_idx * stride_wk, + mask=route_active, + other=0.0, + ) base_acc += base rank_acc += routed_rank * weight @@ -366,6 +438,7 @@ def fused_base_mapped_shared_lora_reduce( output: torch.Tensor, routed_scaling_factor: float, *, + topk_ids: torch.Tensor | None = None, block_k: int, ) -> None: """Fused decode tail with a per-token shared-B adapter slot.""" @@ -417,6 +490,11 @@ def fused_base_mapped_shared_lora_reduce( if routed_base.shape[0] == 0: return + if topk_ids is not None: + assert topk_ids.shape == topk_weights.shape + assert topk_ids.device == routed_rank.device + assert topk_ids.dtype in (torch.int32, torch.int64) + assert block_k in (32, 64, 128) block_rank = max(16, triton.next_power_of_2(routed_rank.shape[2])) _fused_base_mapped_shared_lora_reduce_kernel[ @@ -425,6 +503,7 @@ def fused_base_mapped_shared_lora_reduce( routed_base, routed_rank, topk_weights, + topk_ids, shared_b_view, token_lora_mapping, output, @@ -439,12 +518,15 @@ def fused_base_mapped_shared_lora_reduce( routed_rank.stride(2), topk_weights.stride(0), topk_weights.stride(1), + topk_ids.stride(0) if topk_ids is not None else 0, + topk_ids.stride(1) if topk_ids is not None else 0, shared_b_view.stride(0), shared_b_view.stride(1), shared_b_view.stride(2), output.stride(0), output.stride(1), TOPK=routed_base.shape[1], + HAS_TOPK_IDS=topk_ids is not None, NUM_SLOTS=shared_b.shape[0], BLOCK_K=block_k, BLOCK_R=block_rank, diff --git a/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py b/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py index 0875364d30..d6d1452f0c 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py @@ -65,6 +65,9 @@ def _moe_lora_expand_add_kernel( num_pid_m = tl.cdiv(num_tokens_post_padded, BLOCK_SIZE_M) num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + if pid >= num_pid_m * num_pid_n: + return + num_pid_in_group = GROUP_SIZE_M * num_pid_n group_id = pid // num_pid_in_group first_pid_m = group_id * GROUP_SIZE_M @@ -72,9 +75,6 @@ def _moe_lora_expand_add_kernel( pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) pid_n = (pid % num_pid_in_group) // group_size_m - if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: - return - offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64) token_mask = offs_token < num_valid_tokens diff --git a/test/registered/kernels/ops/lora/test_moe_lora_routing.py b/test/registered/kernels/ops/lora/test_moe_lora_routing.py index e19ba7f448..59ebb98b49 100644 --- a/test/registered/kernels/ops/lora/test_moe_lora_routing.py +++ b/test/registered/kernels/ops/lora/test_moe_lora_routing.py @@ -252,6 +252,68 @@ class TestMoeLoraRouting(CustomTestCase): for count in counts: self.assertEqual(torch.count_nonzero(count).item(), 0) + def test_full_width_route_domains_across_builders_and_graph_replay(self): + """Wide signed IDs retain their validity through every aligned route builder.""" + wide = 2**40 + base_slots = torch.tensor([0, -wide, wide, 1, 0, 0, 1], dtype=torch.int64) + base_groups = torch.tensor( + [ + [0, 1], + [0, 1], + [0, 1], + [-wide, wide], + [2**31, -1], + [torch.iinfo(torch.int64).min, torch.iinfo(torch.int64).max], + [2, 3], + ], + dtype=torch.int64, + ) + for groups_per_slot in (1, 4): + for builder, repeats in (("small", 1), ("jit", 64), ("large", 1)): + with self.subTest(groups_per_slot=groups_per_slot, builder=builder): + host_slots = base_slots.repeat(repeats) + host_groups = base_groups.repeat(repeats, 1) + slots = host_slots.to(self.device) + groups = host_groups.to(self.device) + workspace = LoraWorkspace() + workspace.begin_forward(graph_mode=True) + capacity = 256 if builder == "large" else None + + def build(): + return _tensors( + build_route( + slots, + group_ids=groups, + groups_per_slot=groups_per_slot, + max_loras=2, + block_size=16, + workspace=workspace, + tensor_prefix=f"fullwidth:{builder}", + capacity=capacity, + ) + ) + + graph, result = self._capture(build) + for state in ("initial", "inactive", "recovery"): + next_slots = host_slots.clone() + next_groups = host_groups.clone() + if state == "inactive": + next_slots.fill_(-wide) + next_groups.fill_(-wide) + slots.copy_(next_slots) + groups.copy_(next_groups) + for tensor in result: + tensor.fill_(-7) + graph.replay() + expected = self._expected( + next_slots, next_groups, groups_per_slot, 2 + ) + self._assert_layout( + result, expected, 16, sentinel_last=builder == "large" + ) + self.assertEqual(slots.cpu().tolist(), next_slots.tolist()) + self.assertEqual(groups.cpu().tolist(), next_groups.tolist()) + def test_parallel_workspace_joins_reused_graph_regions(self): workspace = LoraWorkspace() source = torch.full((64,), 3.0, device=self.device) diff --git a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py index 5cbfbae6ec..cb8874eb96 100644 --- a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py +++ b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py @@ -1069,6 +1069,7 @@ def _check_marlin_inactive_routes( max_lora_rank=rank, has_active_lora=True, experts_shared_outer_loras=True, + adapter_enabled=torch.ones(1, device="cuda", dtype=torch.int32), gate_up_lora_a_weights=x.new_zeros((1, 1, 2 * rank, hidden)), gate_up_lora_b_weights=x.new_zeros( (1, experts, 2 * intermediate, rank) @@ -1091,9 +1092,11 @@ def _check_marlin_inactive_routes( if runner_kind != "fused": dispatch = StandardDispatchOutput(x, None, routing) if runner_kind == "stock_lora": - return core.run_from_dispatch( + result = core.run_from_dispatch( dispatch, quant_info, config, hooks ).hidden_states + assert hook_weights[0] is routing.topk_weights + return result return fused_experts_experimental_sgl_marlin_lora( dispatch, quant_info, config, lora_info ).hidden_states @@ -1122,20 +1125,23 @@ def _check_marlin_inactive_routes( activated = torch.nn.functional.silu(gate) * up projected = torch.einsum("mti,mthi->mth", activated, w2_ref[ids]) reference = (projected.float() * weights[:, :, None]).sum(1).to(x.dtype) + caller_module = { + "fused": "sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe", + "stock_lora": "sglang.srt.lora.lora_moe_runner_marlin", + "factored_lora": "sglang.srt.lora.marlin_lora_temp.moe_runner", + }[runner_kind] projections = [] def checked_marlin_gemm(*args, **kwargs): - # Ordinary built-in alignment certifies both projections. + # Built-in alignment certifies both projections, including padded + # graph rows; dropping either certificate restores the table scan. assert kwargs.get("expert_ids_are_sorted", False) projections.append(kwargs["mul_topk_weights"]) return moe_wna16_marlin_gemm(*args, **kwargs) with ( patch.object(zero_copy_context, "get_moe_output", return_value=output), - patch( - "sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe.moe_wna16_marlin_gemm", - checked_marlin_gemm, - ), + patch(f"{caller_module}.moe_wna16_marlin_gemm", checked_marlin_gemm), ): for _ in range(3): run() @@ -1156,9 +1162,8 @@ def _check_marlin_inactive_routes( assert torch.count_nonzero(actual[count:]) == 0 if hook_weights: weights, ids = hook_weights - assert torch.count_nonzero(weights[ids < 0]) == 0 - if runner_kind == "fused": - assert projections == [False, True] * 4 + assert torch.isnan(weights[ids < 0]).all() + assert projections == [False, True] * 4 @pytest.mark.parametrize("has_inactive_prefix_count", [False, True]) @@ -1372,6 +1377,117 @@ def test_marlin_lora_direct_inactive_routes(): assert torch.count_nonzero(gate_output[ids < 0]) == 0 assert torch.count_nonzero(down_output[ids < 0]) == 0 + # A dense oracle for both real adapter slots keeps the selected projection + # independent of the kernels' inactive-address and payload masks. + finite_shared = torch.stack((shared, shared * 2)) + finite_gate_b = torch.cat((gate_b, gate_b * 1.5)) + finite_down_a = torch.cat((down_a, down_a * -0.75)) + finite_activation = torch.randn_like(activation) + dense_gate = ( + torch.einsum( + "smhr,sehir->smehi", + finite_shared.view(2, rows, 2, rank).float(), + finite_gate_b.view(2, experts, 2, intermediate, rank).float(), + ) + .flatten(-2) + .to(gate_output.dtype) + ) + dense_down = torch.einsum( + "mti,seri->smter", + finite_activation.view(rows, topk, intermediate).float(), + finite_down_a.float(), + ).to(down_output.dtype) + slot_shared = finite_shared.clone() + slot_gate_b = finite_gate_b.clone() + slot_down_a = finite_down_a.clone() + route_ids = torch.arange(topk, device="cuda", dtype=torch.int32).repeat(rows, 1) + route_index = torch.arange(topk, device="cuda") + states = ( + ("valid", (0, 1, 0), (1, 1)), + ("disabled", (0, 1, 0), (1, 0)), + ("missing-or-out-of-range", (0, -1, 2), (1, 1)), + ("zero", (0, 1, 0), (1, 1)), + ("recovery", (0, 1, 0), (1, 1)), + ) + for enabled_dtype in (torch.bool, torch.int32, torch.int64): + # The optional metadata accepts a strided one-dimensional view. + enabled = torch.ones(4, device="cuda", dtype=enabled_dtype)[::2] + + def project_enabled(): + direct_decode_gate_expand( + slot_shared, + slot_gate_b, + ids, + slots, + gate_output, + adapter_enabled=enabled, + ) + direct_decode_down_shrink( + activation, + slot_down_a, + ids, + slots, + down_output, + adapter_enabled=enabled, + ) + + ids.copy_(route_ids) + slots.copy_(torch.tensor(states[0][1], device="cuda", dtype=slots.dtype)) + activation.copy_(finite_activation) + project_enabled() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + project_enabled() + for state, mapping, flags in states: + ids.copy_(route_ids) + slots.copy_(torch.tensor(mapping, device="cuda", dtype=slots.dtype)) + enabled.copy_(torch.tensor(flags, device="cuda", dtype=enabled_dtype)) + slot_shared.copy_(finite_shared) + slot_gate_b.copy_(finite_gate_b) + slot_down_a.copy_(finite_down_a) + activation.copy_(finite_activation) + if state == "disabled": + slot_shared[1].fill_(float("nan")) + slot_gate_b[1].fill_(float("nan")) + slot_down_a[1].fill_(float("nan")) + activation.view(rows, topk, intermediate)[1].fill_(float("nan")) + elif state == "missing-or-out-of-range": + slot_shared[:, 1:].fill_(float("nan")) + activation.view(rows, topk, intermediate)[1:].fill_(float("nan")) + elif state == "zero": + ids.fill_(-1) + slot_shared.fill_(float("nan")) + slot_gate_b.fill_(float("nan")) + slot_down_a.fill_(float("nan")) + activation.fill_(float("nan")) + gate_output.fill_(float("nan")) + down_output.fill_(float("nan")) + graph.replay() + expected_gate.zero_() + expected_down.zero_() + active_rows = [] + if state != "zero": + for token, slot in enumerate(mapping): + if 0 <= slot < len(flags) and flags[slot]: + active_rows.append(token) + expected_gate[token] = dense_gate[slot, token, route_index] + expected_down[token] = dense_down[ + slot, token, route_index, route_index + ] + torch.testing.assert_close(gate_output, expected_gate, rtol=0.03, atol=0.01) + torch.testing.assert_close(down_output, expected_down, rtol=0.03, atol=0.01) + assert torch.isfinite(gate_output).all() + assert torch.isfinite(down_output).all() + inactive_rows = [token for token in range(rows) if token not in active_rows] + assert torch.count_nonzero(gate_output[inactive_rows]) == 0 + assert torch.count_nonzero(down_output[inactive_rows]) == 0 + if active_rows: + # Slot zero is a loaded adapter and must remain a real projection. + assert 0 in active_rows + assert torch.count_nonzero(expected_gate[0]) > 0 + assert torch.count_nonzero(expected_down[0]) > 0 + @pytest.mark.parametrize("use_expert_map", [False, True]) @pytest.mark.parametrize("weight_first", [False, True]) diff --git a/test/registered/lora/test_virtual_experts_kernels.py b/test/registered/lora/test_virtual_experts_kernels.py index 8b420eaceb..8163ebea1b 100644 --- a/test/registered/lora/test_virtual_experts_kernels.py +++ b/test/registered/lora/test_virtual_experts_kernels.py @@ -25,6 +25,9 @@ Usage: """ import unittest +from itertools import product +from types import SimpleNamespace +from unittest.mock import patch import torch @@ -32,7 +35,8 @@ from sglang.srt.utils import get_device, is_cuda, is_xpu from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=14, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=14, stage="extra-a", runner_config="1-gpu-small") +# backend-specific: XPU covers Triton sentinels and the torch alignment fallback. register_xpu_ci(est_time=20, suite="stage-a-test-1-gpu-xpu") @@ -46,7 +50,6 @@ from sglang.kernels.ops.lora.moe.virtual_experts import ( _align_block_size_jit, _align_block_size_torch, _fused_virtual_topk_ids, - fused_sanitize_expert_ids, ) @@ -57,48 +60,1018 @@ class TestFusedVirtualTopkIdsPreservesSentinels(CustomTestCase): def setUpClass(cls): cls.device = _require_accelerator_device() + @unittest.skipIf(is_xpu(), "CUDA graph replay required") + def test_capacity_tail_is_not_consumed_on_replay(self): + """Unused routing capacity never contributes, including a zero used count.""" + import sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts as temp + import sglang.kernels.ops.lora.moe.virtual_experts as stock + from sglang.srt.lora.trtllm_lora_temp.specialized_expand import ( + _moe_lora_expand_add_kernel, + ) + + block_m, block_n, capacity_blocks = 16, 16, 9 + routes, width, router_topk = block_m * capacity_blocks, 33, 2 + sorted_ids = torch.zeros(routes, dtype=torch.int32, device=self.device) + expert_ids = torch.full( + (capacity_blocks,), -3, dtype=torch.int32, device=self.device + ) + count = torch.zeros(1, dtype=torch.int32, device=self.device) + order = torch.arange(routes, dtype=torch.int32, device=self.device) + route_values = (torch.arange(routes, device=self.device) % 7).half() / 8 + scores = torch.full((routes,), 0.5, device=self.device, dtype=torch.float16) + + def publish(blocks): + count.fill_(blocks * block_m) + sorted_ids.zero_() + sorted_ids[: blocks * block_m].copy_(order[: blocks * block_m]) + expert_ids.fill_(-3) + expert_ids[:blocks].fill_(0) + if blocks: + expert_ids[0] = -1 + + variants = ( + [ + ("stock-shrink", stock._moe_lora_shrink_splitk_kernel, split_k) + for split_k in (1, 2) + ] + + [ + ("temp-shrink", temp._moe_lora_shrink_splitk_kernel, split_k) + for split_k in (1, 2) + ] + + [("expand", _moe_lora_expand_add_kernel, add) for add in (False, True)] + ) + for name, kernel, mode in variants: + with self.subTest(kernel=name, mode=mode): + shrink = name != "expand" + contraction = 64 if shrink else 32 + a = route_values[:, None].expand(routes, contraction).contiguous() + # Negative tail IDs address a poisoned leading record if consumed. + backing = torch.full( + (4, width, contraction), + float("nan"), + dtype=torch.float16, + device=self.device, + ) + weight = backing[3:] + weight.fill_(0.125) + additive = not shrink and mode + output_rows = routes // router_topk if additive else routes + output = torch.empty( + (output_rows, width), dtype=torch.float16, device=self.device + ) + initial = 0.0 if shrink and mode > 1 else 0.75 + baseline = torch.full_like(output, initial) + grid = (capacity_blocks * ((width + block_n - 1) // block_n),) + + def launch(): + if shrink: + kernel[(grid[0] * mode,)]( + a, + weight, + output, + sorted_ids, + expert_ids, + count, + width, + contraction, + routes, + a.stride(0), + a.stride(1), + weight.stride(0), + weight.stride(1), + weight.stride(2), + output.stride(0), + output.stride(1), + top_k=1, + BLOCK_SIZE_M=block_m, + BLOCK_SIZE_N=block_n, + BLOCK_SIZE_K=32, + GROUP_SIZE_M=4, + SPLIT_K=mode, + num_warps=4, + num_stages=1, + ) + else: + kernel[grid]( + a, + weight, + output, + scores, + sorted_ids, + expert_ids, + count, + width, + contraction, + routes, + a.stride(0), + a.stride(1), + weight.stride(0), + weight.stride(1), + weight.stride(2), + output.stride(0), + output.stride(1), + router_topk=router_topk, + MUL_ROUTED_WEIGHT=True, + FUSE_SUM_ALL_REDUCE=mode, + BLOCK_SIZE_M=block_m, + BLOCK_SIZE_N=block_n, + BLOCK_SIZE_R=32, + GROUP_SIZE_M=4, + GATED_A_HALF=0, + BROADCAST_A=False, + num_warps=4, + num_stages=1, + ) + + publish(capacity_blocks) + output.copy_(baseline) + launch() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + launch() + for blocks in (0, 3, 5, capacity_blocks, 0, 3): + publish(blocks) + output.copy_(baseline) + graph.replay() + expected = baseline.clone() + active_end = blocks * block_m + projection = route_values * (contraction * 0.125) + if shrink: + expected[block_m:active_end] = projection[ + block_m:active_end, None + ] + elif additive: + delta = torch.zeros(routes, device=self.device).half() + delta[block_m:active_end] = projection[block_m:active_end] * 0.5 + expected += delta.view(-1, router_topk).sum(1)[:, None] + elif blocks: + expected[:block_m] = 0 + expected[block_m:active_end] = ( + projection[block_m:active_end, None] * 0.5 + ) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + @unittest.skipIf(is_xpu(), "CUDA paired routing and graph replay required") + def test_paired_routing_matches_staged_routes_and_replays(self): + """Both per-expert stages share raw routes, including distinct tuned blocks.""" + import weakref + + import sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts as temp + from sglang.srt.layers.moe.moe_runner.triton_utils import ( + fused_moe_triton_config, + ) + + slots, experts, rank, hidden, width, topk = 3, 3, 8, 65, 33, 2 + # Strided payload metadata must not force flatten/copy during routing. + a = torch.empty( + (slots, experts, rank, hidden * 2), device=self.device, dtype=torch.float16 + )[..., ::2] + b = torch.empty( + (slots, experts, width * 2, rank), device=self.device, dtype=a.dtype + )[:, :, ::2, :] + shared_a = torch.empty( + (slots, 1, rank, hidden), device=self.device, dtype=a.dtype + ) + shared_b = torch.empty( + (slots, 1, hidden, rank), device=self.device, dtype=a.dtype + ) + + def config(w1_shape, w2_shape, stage_topk, dtype, tokens): + # Gate B and down shared B deliberately select different blocks. + return { + "BLOCK_SIZE_M": 32 if w2_shape[1] == width else 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 16, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + } + + for tokens in (6, 512): + with ( + self.subTest(tokens=tokens), + patch.object( + fused_moe_triton_config, + "try_get_optimal_moe_config", + side_effect=config, + ), + patch.object( + temp.lora_envs.SGLANG_OPT_LORA_SHRINK_TUNE, + "get", + return_value=False, + ), + patch.object( + temp.lora_envs.SGLANG_OPT_LORA_PREFILL_ROUTING_REUSE, + "get", + return_value=True, + ), + ): + x = torch.empty((tokens, hidden), device=self.device, dtype=a.dtype) + ids = torch.arange(tokens * topk, device=self.device).view(tokens, topk) + ids.remainder_(experts) + mapping = torch.arange(tokens, device=self.device).remainder_(slots) + enabled = torch.ones(slots * 2, device=self.device, dtype=torch.int32)[ + ::2 + ] + weights = torch.ones((tokens, topk), device=self.device) + output = torch.empty( + (tokens, topk, width), device=self.device, dtype=a.dtype + ) + + def prepare(cache): + self.assertTrue( + temp._prewarm_per_expert_lora_routes( + x, + a, + b, + ids, + mapping, + cache, + shrink_lora_b=shared_b, + adapter_enabled=enabled, + include_inactive_routes=True, + local_num_experts=experts, + ) + ) + + def check(cache): + valid_slot = (mapping >= 0) & (mapping < slots) + safe_slot = mapping.clamp(0, slots - 1) + mask = valid_slot & (enabled[safe_slot] != 0) + expected = torch.where( + mask[:, None] & (ids >= 0) & (ids < experts), + ids + mapping[:, None] * experts, + -1, + ).flatten() + for key, ( + sorted_ids, + expert_ids, + count, + token_mask, + ) in cache.items(): + block = key[2] + used = int(count[0].item()) + self.assertLessEqual(used, sorted_ids.numel()) + route = sorted_ids[:used].long() + expert = expert_ids[: used // block].repeat_interleave(block) + real = route < ids.numel() + route, expert = route[real], expert[real] + self.assertEqual(route.numel(), ids.numel()) + self.assertEqual(route.unique().numel(), ids.numel()) + actual = torch.empty_like(expected) + actual[route] = expert.to(actual.dtype) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close(token_mask, mask, rtol=0, atol=0) + + raw_refs = [] + remap = temp._fused_virtual_topk_ids + + def observe_raw(*args, **kwargs): + result = remap(*args, **kwargs) + raw_refs.append(weakref.ref(result[0])) + return result + + cache = {} + with patch.object( + temp, "_fused_virtual_topk_ids", side_effect=observe_raw + ): + prepare(cache) + self.assertEqual(len(raw_refs), 1) + self.assertIsNone(raw_refs[0]()) + expected_blocks = {32, 64 if tokens >= 512 else 16} + self.assertEqual({key[2] for key in cache}, expected_blocks) + tables = list(cache.values()) + self.assertIs(tables[0][3], tables[1][3]) + check(cache) + + # The unchanged public staged route path is an independent cache oracle. + reference = {} + common = dict( + output=output, + hidden_states=x, + topk_ids=ids, + topk_weights=weights, + token_lora_mapping=mapping, + routing_cache=reference, + stage="routing", + include_inactive_routes=True, + adapter_enabled=enabled, + local_num_experts=experts, + ) + temp.merged_experts_fused_moe_lora_add( + **common, + lora_a=shared_a, + lora_b=b, + mul_routed_weight=False, + experts_shared_outer_loras_a=True, + experts_shared_outer_loras_b=False, + prewarm_a_routing=False, + ) + temp.merged_experts_fused_moe_lora_add( + **common, + lora_a=a, + lora_b=shared_b, + mul_routed_weight=True, + experts_shared_outer_loras_a=False, + experts_shared_outer_loras_b=True, + prewarm_b_routing=False, + ) + self.assertEqual(set(cache), set(reference)) + check(reference) + + # Enabled identity and inactive-route mode retain their distinct keys. + other_enabled = enabled.clone().zero_() + self.assertTrue( + temp._prewarm_per_expert_lora_routes( + x, + a, + b, + ids, + mapping, + cache, + shrink_lora_b=shared_b, + adapter_enabled=other_enabled, + include_inactive_routes=True, + ) + ) + self.assertTrue( + temp._prewarm_per_expert_lora_routes( + x, + a, + b, + ids, + mapping, + cache, + shrink_lora_b=shared_b, + adapter_enabled=enabled, + include_inactive_routes=False, + ) + ) + self.assertEqual(len(cache), 6) + for key, value in cache.items(): + if key[4] == id(other_enabled): + self.assertFalse(bool(value[3].any())) + # Global-ID EP keeps the existing staged/merged provider path. + untouched = {} + self.assertFalse( + temp._prewarm_per_expert_lora_routes( + x, + a, + b, + ids, + mapping, + untouched, + shrink_lora_b=shared_b, + local_num_experts=experts - 1, + ) + ) + self.assertEqual(untouched, {}) + self.assertTrue( + temp._prewarm_per_expert_lora_routes( + x[:0], + a, + b, + ids[:0], + mapping[:0], + untouched, + shrink_lora_b=shared_b, + adapter_enabled=enabled, + ) + ) + self.assertEqual(untouched, {}) + + if tokens != 6: + continue + # Graph replays mutate borrowed contents, not identities or cache keys. + initial_ids, initial_mapping = ids.clone(), mapping.clone() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + replay_cache = {} + prepare(replay_cache) + for state in ("partial", "disabled", "missing", "zero", "recovery"): + ids.copy_(initial_ids) + mapping.copy_(initial_mapping) + enabled.fill_(1) + if state == "partial": + ids[0, 0] = -2 + ids[1, 1] = experts + elif state == "disabled": + enabled.zero_() + elif state == "missing": + mapping[0] = -1 + mapping[1] = slots + 1 + elif state == "zero": + ids.fill_(-1) + saved_ids, saved_mapping = ids.clone(), mapping.clone() + graph.replay() + check(replay_cache) + torch.testing.assert_close(ids, saved_ids, rtol=0, atol=0) + torch.testing.assert_close(mapping, saved_mapping, rtol=0, atol=0) + + @unittest.skipIf(is_xpu(), "CUDA staged virtual-expert helpers required") + def test_empty_virtual_calls_preserve_stage_results(self): + """Empty route sets preserve add outputs and the staged shrink return contract.""" + import sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts as temp + import sglang.kernels.ops.lora.moe.virtual_experts as stock + from sglang.srt.layers.moe.moe_runner.triton_utils import ( + fused_moe_triton_config, + ) + + hidden, width, rank = 32, 16, 16 + a = torch.empty((1, 2, rank, hidden), device=self.device, dtype=torch.float16) + b = torch.empty((1, 2, width, rank), device=self.device, dtype=torch.float16) + with patch.object( + fused_moe_triton_config, + "try_get_optimal_moe_config", + side_effect=AssertionError( + "empty routes must return before tuning or routing" + ), + ): + for tokens, topk in ((0, 2), (3, 0)): + with self.subTest(tokens=tokens, topk=topk): + backing = torch.full( + (3, 2, width), 0.75, device=self.device, dtype=torch.float16 + ) + saved = backing.clone() + output = backing[:tokens, :topk] + cache_marker = object() + cache = {"untouched": cache_marker} + kwargs = dict( + output=output, + hidden_states=torch.empty( + (tokens, hidden), device=self.device, dtype=a.dtype + ), + lora_a=a, + lora_b=b, + topk_ids=torch.empty( + (tokens, topk), device=self.device, dtype=torch.int32 + ), + topk_weights=torch.empty( + (tokens, topk), device=self.device, dtype=torch.float32 + ), + token_lora_mapping=torch.empty( + tokens, device=self.device, dtype=torch.int32 + ), + mul_routed_weight=True, + experts_shared_outer_loras_a=False, + experts_shared_outer_loras_b=False, + routing_cache=cache, + ) + self.assertIsNone(stock.merged_experts_fused_moe_lora_add(**kwargs)) + for stage in ("routing", "all", "shrink", "expand"): + for provide_buffer in (False, True): + with self.subTest( + stage=stage, provide_buffer=provide_buffer + ): + buffer = ( + torch.empty( + (tokens, topk, rank), + device=self.device, + dtype=a.dtype, + ) + if provide_buffer + else None + ) + call_kwargs = dict( + kwargs, stage=stage, intermediate_buffer=buffer + ) + if stage == "expand" and buffer is None: + with self.assertRaisesRegex( + AssertionError, "requires intermediate_buffer" + ): + temp.merged_experts_fused_moe_lora_add( + **call_kwargs + ) + continue + result = temp.merged_experts_fused_moe_lora_add( + **call_kwargs + ) + if stage == "shrink": + self.assertEqual(result.shape, (tokens, topk, rank)) + self.assertEqual(result.dtype, a.dtype) + self.assertEqual(result.device, a.device) + if buffer is not None: + self.assertIs(result, buffer) + else: + self.assertIsNone(result) + if tokens: + reduced = backing[:, 0] + self.assertIsNone( + temp.merged_experts_fused_moe_lora_add( + **dict(kwargs, output=reduced, fuse_sum_all_reduce=True) + ) + ) + with self.assertRaisesRegex(AssertionError, "invalid stage"): + temp.merged_experts_fused_moe_lora_add( + **dict(kwargs, stage="invalid") + ) + self.assertEqual(cache, {"untouched": cache_marker}) + torch.testing.assert_close(backing, saved, rtol=0, atol=0) + def test_negative_sentinels_preserved(self): - # Mix of valid topk_ids in [0, num_experts), -1 sentinels (typical - # post-EP-dispatch), and a synthetic -2 to ensure the fix doesn't - # depend on the exact -1 value. + """Virtual scratch IDs stay bounded without changing borrowed route metadata.""" + for dtype, max_loras in product((torch.int32, torch.int64), (1, 4)): + with self.subTest(dtype=dtype, max_loras=max_loras): + self._check_remap_domain(dtype, max_loras) + + def _check_remap_domain(self, dtype, max_loras): + num_experts = 16 + wide = 2**40 if dtype == torch.int64 else 2**30 topk_ids = torch.tensor( [ [3, -1], - [-1, 5], + [-3, 5], [0, 7], [-1, -1], [2, 9], [11, -2], [-1, 4], [6, -1], + [16, 17], + [wide, -wide], + [torch.iinfo(dtype).min, torch.iinfo(dtype).max], + [0, 15], ], - dtype=torch.int32, + dtype=dtype, device=self.device, ) token_lora_mapping = torch.tensor( - [0, 1, 0, 2, -1, 1, 0, 1], dtype=torch.int32, device=self.device + [0, 1, 0, 2, -1, 1, 0, 1, 0, 1, 4, 5], + dtype=torch.int32, + device=self.device, ) - num_experts = 16 - max_loras = 4 + original_ids = topk_ids.clone() + original_mapping = token_lora_mapping.clone() + providers = {"base": _fused_virtual_topk_ids} + if is_cuda(): + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( + _fused_virtual_topk_ids as trtllm_virtual_ids, + ) - virtual_ids, _, _ = _fused_virtual_topk_ids( - topk_ids, - token_lora_mapping, - num_experts, - shared_outer=False, - max_loras=max_loras, + providers["trtllm"] = trtllm_virtual_ids + for (provider, remap), shared_outer, use_enabled in product( + providers.items(), (False, True), (False, True) + ): + with self.subTest( + provider=provider, shared_outer=shared_outer, enabled=use_enabled + ): + enabled = ( + torch.ones(max_loras * 2, device=self.device, dtype=torch.int32)[ + ::2 + ] + if use_enabled + else None + ) + if enabled is not None and max_loras > 1: + enabled[1] = 0 + + def run(): + return remap( + topk_ids, + token_lora_mapping, + num_experts, + shared_outer=shared_outer, + max_loras=max_loras, + adapter_enabled=enabled, + ) + + virtual_ids, mask, virtual_experts = run() + graph = None + if is_cuda(): + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + virtual_ids, mask, virtual_experts = run() + for state in ("mixed", "disabled", "inactive", "recovered"): + topk_ids.copy_(original_ids) + token_lora_mapping.copy_(original_mapping) + if enabled is not None: + enabled.fill_(1) + if max_loras > 1: + enabled[1] = 0 + if state == "disabled": + if enabled is not None: + token_lora_mapping.zero_() + enabled.zero_() + else: + token_lora_mapping.fill_(-1) + elif state == "inactive": + topk_ids.fill_(-1) + saved_ids, saved_mapping = ( + topk_ids.clone(), + token_lora_mapping.clone(), + ) + if graph is None: + virtual_ids, mask, virtual_experts = run() + else: + graph.replay() + expected = torch.full_like(topk_ids, -1) + for row, slot in enumerate(saved_mapping.cpu().tolist()): + active = 0 <= slot < max_loras and ( + enabled is None or bool(enabled[slot]) + ) + self.assertEqual(bool(mask[row]), active) + for route, expert in enumerate(saved_ids[row].cpu().tolist()): + if ( + active + and expert >= 0 + and (shared_outer or expert < num_experts) + ): + expected[row, route] = ( + slot + if shared_outer + else expert + slot * num_experts + ) + torch.testing.assert_close(virtual_ids, expected, rtol=0, atol=0) + self.assertTrue( + bool( + ( + (virtual_ids == -1) + | ((virtual_ids >= 0) & (virtual_ids < virtual_experts)) + ).all() + ) + ) + torch.testing.assert_close(topk_ids, saved_ids, rtol=0, atol=0) + torch.testing.assert_close( + token_lora_mapping, saved_mapping, rtol=0, atol=0 + ) + + @unittest.skipUnless(is_cuda(), "Merged LoRA alignment requires CUDA") + def test_shared_outer_merged_alignment_skips_original_negative_routes(self): + """Shared-outer blocks contain exactly live original slots after remapping.""" + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.moe_lora_merged_align import ( + moe_lora_merged_align, ) - # Every negative input must stay negative (and equal) in the output. - for m in range(topk_ids.shape[0]): - for k in range(topk_ids.shape[1]): - base = topk_ids[m, k].item() - if base < 0: + topk_ids = torch.tensor( + [[-1, 2, 0], [-1, -1, -1], [1, -3, 3]], + device=self.device, + dtype=torch.int32, + ) + live_ids = topk_ids.clone() + mapping = torch.tensor([0, 1, 2], device=self.device, dtype=torch.int32) + block_size = 16 + for fuse_scatter in (False, True): + for state in ("mixed", "idle", "remapped"): + with self.subTest(fuse_scatter=fuse_scatter, state=state): + topk_ids.copy_(live_ids) + mapping.copy_( + torch.tensor( + [2, 0, 1] if state == "remapped" else [0, 1, 2], + device=self.device, + dtype=torch.int32, + ) + ) + if state == "idle": + topk_ids.fill_(-1) + original_ids = topk_ids.clone() + original_mapping = mapping.clone() + sorted_ids, experts, count, _, _ = moe_lora_merged_align( + topk_ids, + mapping, + num_experts=4, + shared_outer=True, + max_loras=3, + block_size=block_size, + do_skip=True, + fuse_scatter=fuse_scatter, + return_inactive_prefix_count=True, + ) + self.assertEqual(count.shape, (2,)) + self.assertEqual(count.data_ptr() % 8, 0) + self.assertEqual(int(count[1]), 0) + used = int(count[0]) + self.assertGreaterEqual(used, 0) + self.assertEqual(used % block_size, 0) + self.assertLessEqual(used, sorted_ids.numel()) + blocks = sorted_ids[:used].view(-1, block_size) + self.assertTrue( + bool(((blocks >= 0) & (blocks <= topk_ids.numel())).all()) + ) + emitted = blocks[blocks < topk_ids.numel()] + expected = torch.nonzero(topk_ids.flatten() >= 0).flatten() self.assertEqual( - virtual_ids[m, k].item(), - base, - f"negative sentinel at ({m},{k}) was remapped: " - f"{base} -> {virtual_ids[m, k].item()}", + sorted(emitted.cpu().tolist()), expected.cpu().tolist() + ) + for index, block in enumerate(blocks): + flat_ids = block[block < topk_ids.numel()].long() + tokens = flat_ids // topk_ids.shape[1] + self.assertTrue(bool((topk_ids.flatten()[flat_ids] >= 0).all())) + self.assertTrue( + bool((original_mapping[tokens] == experts[index]).all()) + ) + torch.testing.assert_close(topk_ids, original_ids, rtol=0, atol=0) + torch.testing.assert_close( + mapping, original_mapping, rtol=0, atol=0 + ) + topk_ids.copy_(live_ids) + _, experts, count, _, _ = moe_lora_merged_align( + topk_ids, + mapping, + num_experts=4, + shared_outer=True, + max_loras=3, + block_size=block_size, + do_skip=False, + fuse_scatter=fuse_scatter, + return_inactive_prefix_count=True, + ) + expected_prefix = (int((topk_ids < 0).sum()) + block_size - 1) // block_size + self.assertEqual(int(count[1]), expected_prefix) + self.assertTrue(bool((experts[:expected_prefix] == -1).all())) + ep_ids = torch.tensor( + [[-1, 4, 5], [-3, -1, 4]], device=self.device, dtype=torch.int32 + ) + ep_mapping = torch.zeros(2, device=self.device, dtype=torch.int32) + sorted_ep, ep_experts, ep_count, _, _ = moe_lora_merged_align( + ep_ids, + ep_mapping, + num_experts=8, + shared_outer=False, + max_loras=1, + block_size=block_size, + local_expert_offset=4, + local_num_experts=2, + do_skip=False, + compact=True, + fuse_scatter=fuse_scatter, + return_inactive_prefix_count=True, + ) + self.assertEqual(int(ep_count[1]), 1) + used_blocks = sorted_ep[: int(ep_count[0])].view(-1, block_size) + for index, block in enumerate(used_blocks): + live = block[block < ep_ids.numel()].long() + actual = ep_ids.flatten()[live] + expected = torch.where(actual < 0, -1, actual) + self.assertTrue(bool((expected == ep_experts[index]).all())) + + @unittest.skipUnless(is_cuda(), "Merged LoRA alignment requires CUDA") + def test_merged_alignment_raw_domains_on_replay(self): + """Raw integer extremes and missing adapters never become real weight IDs.""" + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.moe_lora_merged_align import ( + moe_lora_merged_align, + ) + + num_experts, slots, block_size = 4, 3, 8 + for dtype, shared_outer, use_enabled, fuse_scatter in product( + (torch.int32, torch.int64), (False, True), (False, True), (False, True) + ): + with self.subTest( + dtype=dtype, + shared=shared_outer, + enabled=use_enabled, + fused=fuse_scatter, + ): + wide = 2**40 if dtype == torch.int64 else 2**30 + ids = torch.tensor( + [ + [0, 1], + [-3, 2], + [4, 5], + [wide, 0], + [0, 1], + [2, -1], + [torch.iinfo(dtype).min, torch.iinfo(dtype).max], + ], + device=self.device, + dtype=dtype, + ) + mapping = torch.tensor( + [0, 1, 2, 0, -1, slots, 1], device=self.device, dtype=torch.int32 + ) + original_ids, original_mapping = ids.clone(), mapping.clone() + enabled = ( + torch.ones(slots * 2, device=self.device, dtype=torch.int32)[::2] + if use_enabled + else None + ) + + def run(): + return moe_lora_merged_align( + ids, + mapping, + num_experts, + shared_outer, + slots, + block_size, + do_skip=False, + fuse_scatter=fuse_scatter, + adapter_enabled=enabled, + return_inactive_prefix_count=True, + ) + + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + sorted_ids, experts, count, adapter_mask, virtual_experts = run() + for state in ("mixed", "disabled", "inactive", "recovered"): + ids.copy_(original_ids) + mapping.copy_(original_mapping) + if enabled is not None: + enabled.fill_(1) + enabled[1] = 0 + if state == "disabled": + mapping.fill_(-1) + if enabled is not None: + enabled.zero_() + elif state == "inactive": + ids.fill_(-1) + saved_ids, saved_mapping = ids.clone(), mapping.clone() + graph.replay() + groups = {} + for row, slot in enumerate(saved_mapping.cpu().tolist()): + active = 0 <= slot < slots and ( + enabled is None or bool(enabled[slot]) + ) + self.assertEqual(bool(adapter_mask[row]), active) + for route, expert in enumerate(saved_ids[row].cpu().tolist()): + virtual = -1 + if ( + active + and expert >= 0 + and (shared_outer or expert < num_experts) + ): + virtual = ( + slot + if shared_outer + else expert + slot * num_experts + ) + groups.setdefault(virtual, []).append( + row * ids.shape[1] + route + ) + expected_total = sum( + ((len(group) + block_size - 1) // block_size) * block_size + for group in groups.values() + ) + expected_prefix = ( + len(groups.get(-1, [])) + block_size - 1 + ) // block_size + self.assertEqual( + count.cpu().tolist(), [expected_total, expected_prefix] + ) + self.assertLessEqual(expected_total, sorted_ids.numel()) + expected_experts, offset = [], 0 + for expert, indices in sorted(groups.items()): + padded = ( + (len(indices) + block_size - 1) // block_size + ) * block_size + actual = sorted_ids[offset : offset + padded].cpu() + self.assertEqual( + sorted(actual[actual < ids.numel()].tolist()), indices + ) + self.assertTrue( + bool((actual[actual >= ids.numel()] == ids.numel()).all()) + ) + expected_experts += [expert] * (padded // block_size) + offset += padded + self.assertEqual( + experts[: expected_total // block_size].cpu().tolist(), + expected_experts, + ) + self.assertTrue( + all( + expert == -1 or 0 <= expert < virtual_experts + for expert in expected_experts + ) + ) + torch.testing.assert_close(ids, saved_ids, rtol=0, atol=0) + torch.testing.assert_close(mapping, saved_mapping, rtol=0, atol=0) + + @unittest.skipUnless(is_cuda(), "Stock virtual LoRA hook fixture requires CUDA") + def test_gate_up_hook_uses_each_stacked_a_projection(self): + """One shrink/expand pairs independent gate/up ranks, including ragged halves.""" + from sglang.kernels.ops.lora.moe import virtual_experts + from sglang.srt.layers.moe.moe_runner.triton_utils import override_config + from sglang.srt.lora.lora_moe_runners import build_lora_hooks + + tokens, topk, experts, rank, hidden = 3, 2, 2, 16, 64 + dtype = torch.float16 + x = torch.zeros((tokens, hidden), device=self.device, dtype=dtype) + x[:, 0] = torch.arange(1, tokens + 1, device=self.device, dtype=dtype) + ids = torch.tensor( + [[-1, 1], [0, 1], [-1, -1]], device=self.device, dtype=torch.int32 + ) + weights = torch.tensor( + [[float("nan"), 0.5], [0.25, 0.75], [float("nan"), float("nan")]], + device=self.device, + ) + mapping = torch.tensor([0, 1, 0], device=self.device, dtype=torch.int32) + config = { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + } + real_add = virtual_experts.merged_experts_fused_moe_lora_add + # Split-A, common-A and non-gated layouts retain one full-width call. + for (a_parts, b_parts), intermediate in product( + ((2, 2), (1, 2), (1, 1)), (17, 32, 48) + ): + for shared_outer in (False, True): + with self.subTest( + a_parts=a_parts, + b_parts=b_parts, + intermediate=intermediate, + shared_outer=shared_outer, + ): + a = torch.zeros( + (2, 1 if shared_outer else experts, a_parts * rank, hidden), + device=self.device, + dtype=dtype, + ) + b = torch.zeros( + (2, experts, b_parts * intermediate, rank), + device=self.device, + dtype=dtype, + ) + for slot in range(2): + for expert in range(a.shape[1]): + a[slot, expert, 0, 0] = 2 + slot + expert + if a_parts == 2: + a[slot, expert, rank, 0] = 4 + slot + expert + for expert in range(experts): + b[slot, expert, :intermediate, 0] = 5 + slot + expert + if b_parts == 2: + b[slot, expert, intermediate:, 0] = 7 + slot + expert + output = torch.full( + (tokens, topk, b_parts * intermediate), + 0.5, + device=self.device, + dtype=dtype, + ) + safe_ids = ids.clamp_min(0) + a_ids = torch.zeros_like(ids) if shared_outer else safe_ids + selected_a = a[mapping[:, None], a_ids] + selected_b = b[mapping[:, None], safe_ids] + shrunk = torch.einsum( + "mh,mkrh->mkr", x.float(), selected_a.float() + ).to(dtype) + if a_parts == 2: + delta = torch.einsum( + "mkpr,mkpir->mkpi", + shrunk.view(tokens, topk, 2, rank).float(), + selected_b.view( + tokens, topk, 2, intermediate, rank + ).float(), + ).flatten(2) + else: + delta = torch.einsum( + "mkr,mkir->mki", shrunk.float(), selected_b.float() + ) + delta = delta.to(dtype).masked_fill((ids < 0).unsqueeze(-1), 0) + expected = output + delta + info = SimpleNamespace( + max_lora_rank=rank, + has_active_lora=True, + lora_use_virtual_experts=True, + experts_shared_outer_loras=shared_outer, + gate_up_lora_a_weights=a, + gate_up_lora_b_weights=b, + token_lora_mapping=mapping, + adapter_enabled=torch.ones( + 2, device=self.device, dtype=torch.int32 + ), + ) + calls = [] + + def observed_add(**kwargs): + projected = kwargs["output"] + self.assertEqual(projected.stride(-2), output.shape[-1]) + self.assertEqual( + projected.untyped_storage().data_ptr(), + output.untyped_storage().data_ptr(), + ) + calls.append( + ( + projected.data_ptr(), + kwargs["lora_a"].shape[2], + kwargs.get("a_output_slices", 1), + ) + ) + return real_add(**kwargs) + + with ( + override_config(config), + patch( + "sglang.srt.lora.lora_moe_runners.get_is_capture_mode", + return_value=False, + ), + patch.object( + virtual_experts, + "merged_experts_fused_moe_lora_add", + observed_add, + ), + ): + hooks = build_lora_hooks(x, info, ids) + hooks.after_gate_up(x, output, weights, ids) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + self.assertEqual( + calls, [(output.data_ptr(), a_parts * rank, a_parts)] ) def test_positive_topk_remapped_correctly(self): @@ -124,9 +1097,8 @@ class TestFusedVirtualTopkIdsPreservesSentinels(CustomTestCase): expected = base + max(lora, 0) * num_experts self.assertEqual(virtual_ids[m, k].item(), expected) - def test_no_lora_token_does_not_shift_base(self): - """`token_lora_mapping[m] == -1` (no LoRA) keeps `safe_lora=0`, - so positive bases pass through unchanged and the row mask is False.""" + def test_no_lora_token_has_no_virtual_expert(self): + """A missing adapter never occupies a virtual expert, including slot zero.""" topk_ids = torch.tensor([[3, 5]], dtype=torch.int32, device=self.device) token_lora_mapping = torch.tensor([-1], dtype=torch.int32, device=self.device) num_experts = 16 @@ -134,8 +1106,10 @@ class TestFusedVirtualTopkIdsPreservesSentinels(CustomTestCase): virtual_ids, mask, _ = _fused_virtual_topk_ids( topk_ids, token_lora_mapping, num_experts, False, max_loras=4 ) - self.assertEqual(virtual_ids[0, 0].item(), 3) - self.assertEqual(virtual_ids[0, 1].item(), 5) + torch.testing.assert_close(virtual_ids, torch.full_like(virtual_ids, -1)) + torch.testing.assert_close( + topk_ids, torch.tensor([[3, 5]], device=self.device, dtype=torch.int32) + ) self.assertFalse(bool(mask[0].item())) @@ -154,6 +1128,15 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): def _align(self, topk_ids, block_size, num_experts): raise NotImplementedError + def _align_used(self, topk_ids, block_size, num_experts): + sorted_ids, experts, count = self._align(topk_ids, block_size, num_experts) + used = int(count[0]) + self.assertGreaterEqual(used, 0) + self.assertEqual(used % block_size, 0) + self.assertLessEqual(used, sorted_ids.numel()) + self.assertLessEqual(used // block_size, experts.numel()) + return sorted_ids, experts[: used // block_size], count + @staticmethod def _assigned_experts(expert_ids: torch.Tensor) -> list: """Return the list of real expert ids assigned to blocks (filtering @@ -178,7 +1161,7 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): device=self.device, ) - _, expert_ids, _ = self._align(topk_ids, block_size, num_experts) + _, expert_ids, _ = self._align_used(topk_ids, block_size, num_experts) self._assert_only_real_or_sentinel(expert_ids, num_experts) assigned = set(self._assigned_experts(expert_ids)) @@ -195,7 +1178,7 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): device=self.device, ) - _, expert_ids, _ = self._align(topk_ids, block_size, num_experts) + _, expert_ids, _ = self._align_used(topk_ids, block_size, num_experts) self._assert_only_real_or_sentinel(expert_ids, num_experts) assigned = self._assigned_experts(expert_ids) @@ -214,7 +1197,7 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): device=self.device, ) - _, expert_ids, _ = self._align(topk_ids, block_size, num_experts) + _, expert_ids, _ = self._align_used(topk_ids, block_size, num_experts) self._assert_only_real_or_sentinel(expert_ids, num_experts) assigned = self._assigned_experts(expert_ids) @@ -241,7 +1224,7 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): device=self.device, ) - _, expert_ids, _ = self._align(topk_ids, block_size, num_experts) + _, expert_ids, _ = self._align_used(topk_ids, block_size, num_experts) self._assert_only_real_or_sentinel(expert_ids, num_experts) assigned = self._assigned_experts(expert_ids) @@ -255,7 +1238,7 @@ class _AlignBlockSizeSentinelBucketBase(CustomTestCase): block_size = 16 topk_ids = torch.empty((0, 2), dtype=torch.int32, device=self.device) - sorted_token_ids, expert_ids, num_post_padded = self._align( + sorted_token_ids, expert_ids, num_post_padded = self._align_used( topk_ids, block_size, num_experts ) @@ -278,15 +1261,10 @@ class TestAlignBlockSizeTorchSentinelBucket(_AlignBlockSizeSentinelBucketBase): "on that platform.", ) class TestAlignBlockSizeJitSentinelBucket(_AlignBlockSizeSentinelBucketBase): - """Test the CUDA JIT kernel path (with fused_sanitize_expert_ids, as in - production).""" + """Test raw-ID CUDA alignment without a post-hoc expert-ID sanitizer.""" def _align(self, topk_ids, block_size, num_experts): - sorted_token_ids, expert_ids, num_tokens_post_padded = _align_block_size_jit( - topk_ids, block_size, num_experts - ) - expert_ids = fused_sanitize_expert_ids(expert_ids, num_experts) - return sorted_token_ids, expert_ids, num_tokens_post_padded + return _align_block_size_jit(topk_ids, block_size, num_experts) if __name__ == "__main__": diff --git a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py index 85d2d16417..054f22fda3 100644 --- a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py +++ b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py @@ -97,6 +97,14 @@ def test_cuda_only_kernel_imports_resolve(): ) imports: list[tuple[str, str]] = [] + # The staged virtual helper is imported inside the entry, outside _is_cuda. + for node in ast.walk(tree): + if ( + isinstance(node, ast.ImportFrom) + and node.module + == "sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts" + ): + imports.extend((node.module, alias.name) for alias in node.names) for node in ast.walk(guarded[0]): assert not isinstance(node, ast.Import), ( "plain `import X` inside the `if _is_cuda:` block is not covered by " @@ -180,17 +188,24 @@ def _run_marlin_flow( ep: bool = False, slots: int = 1, rank: int = 1, + paired_prewarm: bool = True, ): """Run the real runner against recording fakes and return what it did (scheduling, stream ownership, buffer identity -- never kernel numerics).""" calls = SimpleNamespace( merged=[], + paired_prewarm=[], fused_tails=0, marlin_is_ep=[], + marlin_sorted=[], + marlin_inactive_prefix=[], marlin_output_ids=[], marlin_output_initialized=[], + marlin_zero_inactive_output=[], align_num_experts=[], + align_inactive_prefix=[], + align_maps_invalid=[], schedule=[], zeroed=[], event_records=[], @@ -258,6 +273,46 @@ def _run_marlin_flow( return result monkeypatch.setattr(torch, "empty", poisoned_empty) + original_new_empty = torch.Tensor.new_empty + + def tracked_new_empty(tensor, size, *args, **kwargs): + result = original_new_empty(tensor, size, *args, **kwargs) + calls.schedule.append( + ("allocate", stream_state["current"].name, tuple(size), id(result)) + ) + return result + + monkeypatch.setattr(torch.Tensor, "new_empty", tracked_new_empty) + + def prewarm_per_expert_lora_routes( + hidden, + shrink_a, + expand_b, + ids, + mapping, + routing_cache, + *, + shrink_lora_b, + adapter_enabled, + include_inactive_routes, + local_expert_offset, + local_num_experts, + ): + assert hidden is hidden_states + assert shrink_a is lora_info.down_lora_a_weights + assert expand_b is lora_info.gate_up_lora_b_weights + assert shrink_lora_b is lora_info.down_lora_b_weights + assert ids is topk_ids + assert mapping is lora_info.token_lora_mapping + assert isinstance(routing_cache, dict) + assert adapter_enabled is lora_info.adapter_enabled + assert include_inactive_routes is True + assert local_expert_offset == 0 + assert local_num_experts == num_experts + assert stream_state["current"] is main_stream + calls.paired_prewarm.append(paired_prewarm) + calls.schedule.append(("paired_prewarm", "main", paired_prewarm)) + return paired_prewarm def merged_experts_fused_moe_lora_add(**kwargs): intermediate = kwargs.get("intermediate_buffer") @@ -272,8 +327,11 @@ def _run_marlin_flow( "intermediate_id": buffer_id, "shared_b": kwargs["experts_shared_outer_loras_b"], "zero_intermediate": kwargs.get("zero_intermediate", False), + "include_inactive_routes": kwargs["include_inactive_routes"], } ) + assert kwargs["adapter_enabled"] is lora_info.adapter_enabled + assert kwargs["include_inactive_routes"] is True _stub_module( monkeypatch, @@ -285,6 +343,7 @@ def _run_marlin_flow( monkeypatch, "sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts", merged_experts_fused_moe_lora_add=merged_experts_fused_moe_lora_add, + _prewarm_per_expert_lora_routes=prewarm_per_expert_lora_routes, ) _stub_module( monkeypatch, @@ -299,6 +358,12 @@ def _run_marlin_flow( def fake_align(_topk_ids, _block_size, num_experts, **_kwargs): calls.align_num_experts.append(num_experts) + calls.align_inactive_prefix.append( + _kwargs.get("return_inactive_prefix_count", False) + ) + calls.align_maps_invalid.append( + _kwargs.get("map_invalid_experts_to_sentinel", True) + ) return ( torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.int32), @@ -307,6 +372,13 @@ def _run_marlin_flow( def fake_marlin_gemm(_x, output, *_args, **kwargs): calls.marlin_is_ep.append(kwargs["is_ep"]) + calls.marlin_sorted.append(kwargs.get("expert_ids_are_sorted", False)) + calls.marlin_inactive_prefix.append( + kwargs.get("has_inactive_prefix_count", False) + ) + calls.marlin_zero_inactive_output.append( + kwargs.get("zero_inactive_output", False) + ) calls.marlin_output_ids.append(id(output)) calls.marlin_output_initialized.append( bool(torch.isfinite(output).all() and torch.count_nonzero(output) == 0) @@ -318,7 +390,8 @@ def _run_marlin_flow( output.fill_(0) return output - def fake_silu_and_mul_add_delta(_x, _delta, output): + def fake_silu_and_mul_add_delta(_x, _delta, output, **kwargs): + assert not kwargs, "Completed private projections use the original activation" calls.schedule.append(("activation", stream_state["current"].name)) output.fill_(0) @@ -332,7 +405,9 @@ def _run_marlin_flow( *, block_m, block_k, + topk_ids, ): + assert topk_ids is dispatch_output.topk_output.topk_ids calls.fused_tails += 1 calls.schedule.append( ("fused_tail", stream_state["current"].name, block_m, block_k) @@ -402,6 +477,7 @@ def _run_marlin_flow( else torch.arange(tokens, dtype=torch.int32).remainder(slots) ), experts_shared_outer_loras=True, + adapter_enabled=torch.full((slots,), active_lora, dtype=torch.int32), ) runner_config = SimpleNamespace( activation="silu", @@ -422,14 +498,19 @@ def _run_marlin_flow( def test_ep_uses_local_alignment_and_skips_nonlocal_marlin_blocks(monkeypatch): - """EP alignment stays local and the down accumulator starts finite and zero.""" + """EP stays local and delegates inactive output writes to the down GEMM.""" calls = _run_marlin_flow(monkeypatch, tokens=16, ep=True, slots=2, rank=128) assert calls.align_num_experts == [1] assert calls.marlin_is_ep == [True, True] + assert calls.marlin_sorted == [False, False] + assert calls.align_inactive_prefix == [False] + assert calls.align_maps_invalid == [False] + assert calls.marlin_inactive_prefix == [False, False] - assert calls.marlin_output_initialized[1] + assert calls.marlin_zero_inactive_output == [True, True] + assert not any(calls.marlin_output_initialized) down_stage = [ call for call in calls.merged if call["stage"] == "all" and call["shared_b"] @@ -438,6 +519,45 @@ def test_ep_uses_local_alignment_and_skips_nonlocal_marlin_blocks(monkeypatch): assert down_stage[0]["zero_intermediate"] is True +@pytest.mark.parametrize("paired_prewarm", [False, True]) +def test_factored_prewarm_runs_on_main_before_rank_buffers_and_side_fork( + monkeypatch, paired_prewarm +): + calls = _run_marlin_flow( + monkeypatch, + tokens=16, + two_stream=True, + capture=True, + paired_prewarm=paired_prewarm, + ) + assert calls.paired_prewarm == [paired_prewarm] + paired_index = calls.schedule.index(("paired_prewarm", "main", paired_prewarm)) + first_side_wait = next( + index + for index, item in enumerate(calls.schedule) + if item[:2] == ("wait_stream", "side") + ) + activation_index = next( + index for index, item in enumerate(calls.schedule) if item[0] == "activation" + ) + rank_allocations = [ + (index, item) + for index, item in enumerate(calls.schedule) + if item[0] == "allocate" and item[2] in ((16, 2), (16, 2, 1)) + ] + assert len(rank_allocations) == 2 + assert all(item[1] == "main" for _, item in rank_allocations) + assert paired_index < min(index for index, _ in rank_allocations) + assert paired_index < first_side_wait < activation_index + routing = [call for call in calls.merged if call["stage"] == "routing"] + assert len(routing) == (0 if paired_prewarm else 2) + assert all(call["stream"] == "main" for call in routing) + assert [call["stage"] for call in calls.merged if call["stage"] != "routing"] == [ + "expand", + "shrink", + ] + + def test_factored_decode_captured_two_stream_schedule_ownership_and_event_lifetime( monkeypatch, ): @@ -454,6 +574,11 @@ def test_factored_decode_captured_two_stream_schedule_ownership_and_event_lifeti base_mapping=True, ) + assert calls.marlin_sorted == [True, True] + assert calls.align_inactive_prefix == [True] + assert calls.align_maps_invalid == [False] + assert calls.marlin_inactive_prefix == [True, True] + # Capture records the LoRA stages even with no live adapter. shrinks = [call for call in calls.merged if call["stage"] == "shrink"] assert len(shrinks) == 1 diff --git a/test/registered/unit/lora/test_experimental_sgl_marlin_shared_outer_reduce.py b/test/registered/unit/lora/test_experimental_sgl_marlin_shared_outer_reduce.py index a39349fde1..6b349597e2 100644 --- a/test/registered/unit/lora/test_experimental_sgl_marlin_shared_outer_reduce.py +++ b/test/registered/unit/lora/test_experimental_sgl_marlin_shared_outer_reduce.py @@ -9,7 +9,7 @@ import torch from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=8, stage="extra-a", runner_config="1-gpu-small") _CUDA_BF16_AVAILABLE = bool( @@ -23,16 +23,249 @@ _RTOL = 1.6e-2 _ATOL = 1e-3 +@pytest.mark.skipif(not _CUDA_BF16_AVAILABLE, reason="CUDA BF16 required") +@pytest.mark.parametrize("num_slots", [1, 3]) +def test_gate_delta_activation_matches_two_projection_reference(num_slots): + """Gate/up deltas use independent rank slices, including adapter zero.""" + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( + merged_experts_fused_moe_lora_add, + ) + from sglang.srt.layers.moe.moe_runner.triton_utils import override_config + from sglang.srt.lora.marlin_lora_temp.activation import silu_and_mul_add_delta + + device, dtype = "cuda", torch.bfloat16 + tokens, topk, experts, rank, hidden, width = 6, 2, 2, 8, 32, 144 + producer_ids = torch.tensor([[0, 1]] * tokens, device=device, dtype=torch.int32) + producer_mapping = torch.tensor( + [0, 1, 2, 0, 1, 0], device=device, dtype=torch.int32 + ).remainder(num_slots) + a = torch.zeros((num_slots, 1, 2 * rank, hidden), device=device, dtype=dtype) + b = torch.zeros((num_slots, experts, 2 * width, rank), device=device, dtype=dtype) + for slot in range(min(2, num_slots)): + a[slot, 0, 0, 0], a[slot, 0, rank, 0] = 2 + slot, 4 + slot + b[slot, :, :width, 0], b[slot, :, width:, 0] = 3 + slot, 5 + slot + hidden_states = torch.zeros((tokens, hidden), device=device, dtype=dtype) + hidden_states[:, 0] = 0.125 + delta = torch.full( + (tokens, topk, 2 * width), float("nan"), device=device, dtype=dtype + ) + weights = torch.ones((tokens, topk), device=device) + config = dict( + BLOCK_SIZE_M=16, + BLOCK_SIZE_N=32, + BLOCK_SIZE_K=32, + GROUP_SIZE_M=1, + num_warps=4, + num_stages=2, + ) + with override_config(config): + merged_experts_fused_moe_lora_add( + output=delta, + hidden_states=hidden_states, + lora_a=a, + lora_b=b, + topk_ids=producer_ids, + topk_weights=weights, + token_lora_mapping=producer_mapping, + mul_routed_weight=False, + experts_shared_outer_loras_a=True, + experts_shared_outer_loras_b=False, + fuse_add_to_output=False, + use_direct_expand_add=True, + ) + shrunk = torch.einsum( + "mh,mrh->mr", hidden_states.float(), a[producer_mapping.long(), 0].float() + ) + selected_b = b[producer_mapping[:, None].long(), producer_ids.long()] + expected_delta = ( + torch.einsum( + "mpr,mkpir->mkpi", + shrunk.view(tokens, 2, rank), + selected_b.float().view(tokens, topk, 2, width, rank), + ) + .flatten(2) + .to(dtype) + ) + torch.testing.assert_close(delta, expected_delta, rtol=0, atol=0) + assert bool((delta[0] != 0).all()), "valid adapter0 must be expanded" + + base = torch.full_like(delta, 0.5) + output = torch.empty((tokens * topk, width), device=device, dtype=dtype) + silu_and_mul_add_delta(base.view(-1, 2 * width), delta.view(-1, 2 * width), output) + gate = base[..., :width].float() + expected_delta[..., :width].float() + up = base[..., width:].float() + expected_delta[..., width:].float() + torch.testing.assert_close( + output.view(tokens, topk, width), + (torch.nn.functional.silu(gate) * up).to(dtype), + rtol=_RTOL, + atol=_ATOL, + ) + + +@pytest.mark.skipif(not _CUDA_BF16_AVAILABLE, reason="CUDA BF16 required") +@pytest.mark.parametrize("shared_b", [False, True]) +@pytest.mark.parametrize("slots", [1, 3]) +def test_private_delta_projection_writes_inactive_routes_on_replay(shared_b, slots): + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( + merged_experts_fused_moe_lora_add, + ) + from sglang.srt.layers.moe.moe_runner.triton_utils import override_config + from sglang.srt.lora.marlin_lora_temp.activation import silu_and_mul_add_delta + + device, dtype = "cuda", torch.bfloat16 + m, k, e, h, n, r = 6, 2, 2, 32, 32, 8 + ids = torch.tensor([[0, 1]] * m, device=device, dtype=torch.int32) + mapping = torch.tensor( + [0, 1, 2, 0, 1, 2], device=device, dtype=torch.int32 + ).remainder(slots) + enabled = torch.ones(slots * 2, device=device, dtype=torch.int32)[::2] + x = torch.zeros((m, h), device=device, dtype=dtype) + x[:, 0] = 0.125 + a = torch.zeros((slots, 1, r, h), device=device, dtype=dtype) + b = torch.zeros((slots, 1 if shared_b else e, 2 * n, r), device=device, dtype=dtype) + for slot in range(slots): + a[slot, 0, 0, 0] = slot + 2 + b[slot, :, :n, 0], b[slot, :, n:, 0] = slot + 3, slot + 5 + weights = torch.ones((m, k), device=device) + delta = torch.empty((m, k, 2 * n), device=device, dtype=dtype) + base = torch.full_like(delta, 0.5) + activated = torch.empty((m * k, n), device=device, dtype=dtype) + config = dict( + BLOCK_SIZE_M=16, + BLOCK_SIZE_N=32, + BLOCK_SIZE_K=32, + GROUP_SIZE_M=1, + num_warps=4, + num_stages=2, + ) + kwargs = dict( + output=delta, + hidden_states=x, + lora_a=a, + lora_b=b, + topk_ids=ids, + topk_weights=weights, + token_lora_mapping=mapping, + mul_routed_weight=False, + experts_shared_outer_loras_a=True, + experts_shared_outer_loras_b=shared_b, + adapter_enabled=enabled, + include_inactive_routes=True, + ) + + def run(): + cache = {} + merged_experts_fused_moe_lora_add( + **kwargs, + routing_cache=cache, + stage="routing", + ) + merged_experts_fused_moe_lora_add( + **kwargs, + routing_cache=cache, + fuse_add_to_output=False, + ) + silu_and_mul_add_delta(base.view(-1, 2 * n), delta.view(-1, 2 * n), activated) + + with override_config(config): + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for state in ("valid", "mixed", "disabled", "all-inactive", "recovered"): + ids.copy_(torch.tensor([[0, 1]] * m, device=device, dtype=ids.dtype)) + mapping.copy_( + torch.tensor( + [0, 1, 2, 0, 1, 2], device=device, dtype=mapping.dtype + ).remainder(slots) + ) + enabled.fill_(1) + if state == "mixed": + ids[0, 0], mapping[1], mapping[2] = -1, -1, slots + if slots > 1: + enabled[1] = 0 + elif state == "disabled": + enabled.zero_() + elif state == "all-inactive": + ids.fill_(-1) + live_route = ids >= 0 + safe_slots = mapping.clamp(0, slots - 1).long() + live_adapter = ( + (mapping >= 0) & (mapping < slots) & (enabled[safe_slots] != 0) + ) + live = live_route & live_adapter[:, None] + weights.fill_(1) + weights.masked_fill_(~live_route, float("nan")) + # Poison both unwritten destinations and inputs that a disabled + # producer must not read, rather than relying on zero parameters. + x.zero_() + x[:, 0] = 0.125 + x[~live_adapter] = float("nan") + delta.fill_(float("nan")) + base.fill_(0.5) + base.masked_fill_(~live_route.unsqueeze(-1), 0) + graph.replay() + expected = torch.zeros_like(delta) + for row in range(m): + for route in range(k): + if bool(live[row, route]): + slot = int(mapping[row]) + expert = 0 if shared_b else int(ids[row, route]) + expected[row, route] = ( + x[row].float() @ a[slot, 0].float().T + ).to(dtype).float() @ b[slot, expert].float().T + torch.testing.assert_close(delta, expected, rtol=0, atol=0) + gate = base[..., :n].float() + expected[..., :n].float() + up = base[..., n:].float() + expected[..., n:].float() + torch.testing.assert_close( + activated.view(m, k, n), + (torch.nn.functional.silu(gate) * up).to(dtype), + rtol=_RTOL, + atol=_ATOL, + ) + if state == "mixed": + added = torch.full_like(delta, 0.5) + merged_experts_fused_moe_lora_add( + **{**kwargs, "output": added}, + fuse_add_to_output=True, + ) + torch.testing.assert_close( + added, (expected.float() + 0.5).to(dtype), rtol=0, atol=0 + ) + # Routing contains no destination identity. Each actual overwrite must + # initialize its own omitted rows even when metadata is reused. + cache = {} + merged_experts_fused_moe_lora_add( + **kwargs, + routing_cache=cache, + stage="routing", + ) + second_output = torch.full_like(delta, float("nan")) + merged_experts_fused_moe_lora_add( + **{**kwargs, "output": second_output}, + routing_cache=cache, + fuse_add_to_output=False, + ) + torch.testing.assert_close(second_output, expected, rtol=0, atol=0) + + def _reference_reduce( routed_base: torch.Tensor, routed_rank: torch.Tensor, topk_weights: torch.Tensor, shared_b: torch.Tensor, routed_scaling_factor: float, + topk_ids: torch.Tensor | None = None, ) -> torch.Tensor: """Materialize the three production operations with their BF16 boundaries.""" operand_dtype = routed_base.dtype + if topk_ids is not None: + inactive = topk_ids < 0 + routed_base = routed_base.masked_fill(inactive.unsqueeze(-1), 0) + routed_rank = routed_rank.masked_fill(inactive.unsqueeze(-1), 0) + topk_weights = topk_weights.masked_fill(inactive, 0) base_sum = ( routed_base.to(torch.float32) .sum(dim=1) @@ -56,10 +289,16 @@ def _mapped_reference_reduce( shared_b: torch.Tensor, token_lora_mapping: torch.Tensor, routed_scaling_factor: float, + topk_ids: torch.Tensor | None = None, ) -> torch.Tensor: """Same reduction with a per-token adapter slot; ``-1`` rows stay base-only.""" dtype = routed_base.dtype + if topk_ids is not None: + inactive = topk_ids < 0 + routed_base = routed_base.masked_fill(inactive.unsqueeze(-1), 0) + routed_rank = routed_rank.masked_fill(inactive.unsqueeze(-1), 0) + topk_weights = topk_weights.masked_fill(inactive, 0) base_sum = routed_base.float().sum(dim=1).mul(routed_scaling_factor).to(dtype) rank_sum = ( (routed_rank.float() * topk_weights.unsqueeze(-1)) @@ -102,6 +341,7 @@ def test_fused_base_shared_lora_reduce_cuda_graph_parity( from sglang.srt.lora.marlin_lora_temp.shared_outer import ( fused_base_shared_lora_reduce, fused_base_shared_lora_reduce_config, + weighted_topk_rank_sum, ) device = torch.device("cuda") @@ -148,11 +388,19 @@ def test_fused_base_shared_lora_reduce_cuda_graph_parity( * 0.05 ) output = torch.empty((num_tokens, hidden_width), device=device, dtype=dtype) + rank_output = torch.empty((num_tokens, rank), device=device, dtype=dtype) + zero_owned_rank = torch.empty_like(routed_rank) + zero_owned_output = torch.empty_like(rank_output) + topk_ids = torch.zeros((num_tokens, topk), device=device, dtype=torch.int32) + clean_base = routed_base.clone() + clean_rank = routed_rank.clone() + clean_weights = topk_weights.clone() # Production launch geometry, so the tuned tiles and their ragged tails run. block_m, block_k = fused_base_shared_lora_reduce_config(num_tokens) - def invoke() -> None: + def invoke(with_route_ids=True) -> None: + route_kwargs = {"topk_ids": topk_ids} if with_route_ids else {} fused_base_shared_lora_reduce( routed_base, routed_rank, @@ -162,7 +410,30 @@ def test_fused_base_shared_lora_reduce_cuda_graph_parity( routed_scaling_factor, block_m=block_m, block_k=block_k, + **route_kwargs, ) + weighted_topk_rank_sum( + routed_rank, + topk_weights, + rank_output, + routed_scaling_factor, + block_m=1 if num_tokens == 1 else 2, + **route_kwargs, + ) + + # Existing callers may omit route IDs when every payload is defined. + invoke(with_route_ids=False) + expected = _reference_reduce( + routed_base, routed_rank, topk_weights, shared_b, routed_scaling_factor + ) + torch.testing.assert_close(output, expected, rtol=_RTOL, atol=_ATOL) + expected_rank = ( + (routed_rank.float() * topk_weights.unsqueeze(-1)) + .sum(dim=1) + .mul(routed_scaling_factor) + .to(dtype) + ) + torch.testing.assert_close(rank_output, expected_rank, rtol=_RTOL, atol=_ATOL) # Specialize and initialize CUDA state off the capture stream. warmup_stream = torch.cuda.Stream() @@ -177,15 +448,48 @@ def test_fused_base_shared_lora_reduce_cuda_graph_parity( with torch.cuda.graph(graph): invoke() - for replay in range(2): + def invoke_zero_owned_rank(): + weighted_topk_rank_sum( + zero_owned_rank, + topk_weights, + zero_owned_output, + routed_scaling_factor, + block_m=1 if num_tokens == 1 else 2, + topk_ids=topk_ids, + inactive_ranks_are_zero=True, + ) + + zero_owned_rank.copy_(routed_rank) + invoke_zero_owned_rank() + torch.cuda.synchronize() + zero_owned_graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(zero_owned_graph): + invoke_zero_owned_rank() + + for replay, route_state in enumerate(("valid", "mixed", "idle", "valid")): # In-place mutation: replay must follow pointers, not capture-time values. - routed_base.mul_(0.75).add_(0.002 * (replay + 1)) - routed_rank.mul_(-0.5).add_(0.001 * (replay + 1)) - topk_weights.copy_(torch.roll(topk_weights, shifts=1, dims=1)) + routed_base.copy_(clean_base).mul_(0.75).add_(0.002 * (replay + 1)) + routed_rank.copy_(clean_rank).mul_(-0.5).add_(0.001 * (replay + 1)) + topk_weights.copy_(torch.roll(clean_weights, shifts=replay, dims=1)) + topk_ids.fill_(0) + if route_state == "mixed": + topk_ids[:, 0] = -1 + topk_ids[::2, 2] = -3 + if num_tokens > 1: + topk_ids[-1].fill_(-1) + elif route_state == "idle": + topk_ids.fill_(-1) + inactive = topk_ids < 0 + routed_base.masked_fill_(inactive.unsqueeze(-1), float("nan")) + routed_rank.masked_fill_(inactive.unsqueeze(-1), float("nan")) + topk_weights.masked_fill_(inactive, float("nan")) shared_b.mul_(0.875).add_(0.0005 * (replay + 1)) output.fill_(float("nan")) + rank_output.fill_(float("nan")) graph.replay() + zero_owned_rank.copy_(routed_rank).masked_fill_(inactive.unsqueeze(-1), 0) + zero_owned_graph.replay() torch.cuda.synchronize() expected = _reference_reduce( @@ -194,8 +498,27 @@ def test_fused_base_shared_lora_reduce_cuda_graph_parity( topk_weights, shared_b, routed_scaling_factor, + topk_ids, ) torch.testing.assert_close(output, expected, rtol=_RTOL, atol=_ATOL) + expected_rank = ( + ( + routed_rank.masked_fill(inactive.unsqueeze(-1), 0).float() + * topk_weights.masked_fill(inactive, 0).unsqueeze(-1) + ) + .sum(dim=1) + .mul(routed_scaling_factor) + .to(dtype) + ) + torch.testing.assert_close(rank_output, expected_rank, rtol=_RTOL, atol=_ATOL) + torch.testing.assert_close(zero_owned_output, rank_output, rtol=0, atol=0) + assert bool(torch.isfinite(output).all()) + assert bool(torch.isfinite(rank_output).all()) + if route_state == "mixed": + assert int(torch.count_nonzero(expected[0])) > 0 + elif route_state == "idle": + assert int(torch.count_nonzero(output)) == 0 + assert int(torch.count_nonzero(rank_output)) == 0 @pytest.mark.skipif( @@ -256,8 +579,13 @@ def test_fused_base_mapped_shared_lora_reduce_cuda_graph_parity( output = torch.empty( (num_tokens, hidden_width), device=device, dtype=torch.bfloat16 ) + topk_ids = torch.zeros((num_tokens, 6), device=device, dtype=torch.int32) + clean_base = routed_base.clone() + clean_rank = routed_rank.clone() + clean_weights = topk_weights.clone() - def invoke() -> None: + def invoke(with_route_ids=True) -> None: + route_kwargs = {"topk_ids": topk_ids} if with_route_ids else {} fused_base_mapped_shared_lora_reduce( routed_base, routed_rank, @@ -267,8 +595,15 @@ def test_fused_base_mapped_shared_lora_reduce_cuda_graph_parity( output, 1.75, block_k=64, + **route_kwargs, ) + invoke(with_route_ids=False) + expected = _mapped_reference_reduce( + routed_base, routed_rank, topk_weights, shared_b, token_lora_mapping, 1.75 + ) + torch.testing.assert_close(output, expected, rtol=_RTOL, atol=_ATOL) + warmup_stream = torch.cuda.Stream() warmup_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(warmup_stream): @@ -281,25 +616,45 @@ def test_fused_base_mapped_shared_lora_reduce_cuda_graph_parity( with torch.cuda.graph(graph): invoke() - token_lora_mapping.copy_((token_lora_mapping + 1).remainder(num_slots)) - if num_tokens > 1: - token_lora_mapping[0] = -1 - routed_base.mul_(0.75) - routed_rank.mul_(-0.5) - shared_b.mul_(0.875) - output.fill_(float("nan")) - graph.replay() - torch.cuda.synchronize() + for replay, route_state in enumerate(("valid", "mixed", "idle", "valid")): + token_lora_mapping.copy_((token_lora_mapping + 1).remainder(num_slots)) + if num_tokens > 1: + token_lora_mapping[0] = -1 + routed_base.copy_(clean_base).mul_(0.75 + 0.05 * replay) + routed_rank.copy_(clean_rank).mul_(-0.5) + topk_weights.copy_(torch.roll(clean_weights, shifts=replay, dims=1)) + topk_ids.fill_(0) + if route_state == "mixed": + topk_ids[:, 0] = -1 + topk_ids[::2, 2] = -3 + if num_tokens > 1: + topk_ids[-1].fill_(-1) + elif route_state == "idle": + topk_ids.fill_(-1) + inactive = topk_ids < 0 + routed_base.masked_fill_(inactive.unsqueeze(-1), float("nan")) + routed_rank.masked_fill_(inactive.unsqueeze(-1), float("nan")) + topk_weights.masked_fill_(inactive, float("nan")) + shared_b.mul_(0.875) + output.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() - expected = _mapped_reference_reduce( - routed_base, - routed_rank, - topk_weights, - shared_b, - token_lora_mapping, - 1.75, - ) - torch.testing.assert_close(output, expected, rtol=_RTOL, atol=_ATOL) + expected = _mapped_reference_reduce( + routed_base, + routed_rank, + topk_weights, + shared_b, + token_lora_mapping, + 1.75, + topk_ids, + ) + torch.testing.assert_close(output, expected, rtol=_RTOL, atol=_ATOL) + assert bool(torch.isfinite(output).all()) + if route_state == "mixed": + assert int(torch.count_nonzero(expected[0])) > 0 + elif route_state == "idle": + assert int(torch.count_nonzero(output)) == 0 if __name__ == "__main__":