From 299a75ac5d962087f8f09e19b23ff43548fc64c8 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:26:15 -0700 Subject: [PATCH] [Refactor] moe: Derive routing inputs from the layer and active forward (#44994) Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com> --- .github/workflows/_pr-test-simulator-gpu.yml | 2 +- .../kernels/aot/csrc/common_extension.cc | 2 +- .../aot/csrc/quantization/gguf/moe.cuh | 20 +- .../aot/csrc/quantization/gguf/moe_vec.cuh | 6 + .../csrc/gemm/marlin_moe/marlin_template.h | 44 +- .../jit/csrc/moe/moe_permute_prepare.cu | 4 +- python/sglang/kernels/ops/moe/__init__.py | 10 + .../ops/moe/fused_moe_triton_kernels.py | 27 +- .../kernels/ops/moe/moe_fused_mul_sum.py | 22 +- .../kernels/ops/moe/moe_wna16_marlin.py | 6 +- .../kernels/ops/moe/shuffle_weighted_sum.py | 74 ++ python/sglang/srt/batch_overlap/operations.py | 4 + .../srt/eplb/expert_location_dispatch.py | 11 +- .../srt/hardware_backend/npu/moe/topk.py | 47 +- .../srt/layers/attention/aiter_backend.py | 16 +- .../srt/layers/attention/aiter_utils.py | 6 +- python/sglang/srt/layers/moe/cutlass_moe.py | 9 +- .../sglang/srt/layers/moe/cutlass_w4a8_moe.py | 11 +- .../moe/fused_moe_triton/fused_marlin_moe.py | 13 +- .../srt/layers/moe/fused_moe_triton/layer.py | 15 +- python/sglang/srt/layers/moe/hash_topk.py | 58 +- python/sglang/srt/layers/moe/mega_moe.py | 15 - .../sglang/srt/layers/moe/mega_moe_flydsl.py | 16 +- python/sglang/srt/layers/moe/topk.py | 178 ++++- .../layers/muse_spark/muse_spark_fp4_moe.py | 12 +- python/sglang/srt/layers/quantization/gguf.py | 54 +- .../sglang/srt/lora/lora_moe_runner_marlin.py | 15 +- .../lora/marlin_lora_temp/direct_decode.py | 10 +- .../srt/lora/marlin_lora_temp/moe_runner.py | 26 +- .../srt/model_executor/cpu_graph_runner.py | 21 +- .../srt/model_executor/forward_batch_info.py | 31 +- .../srt/model_executor/forward_context.py | 21 +- .../sglang/srt/model_executor/model_runner.py | 10 +- .../srt/model_executor/runner/base_runner.py | 4 +- .../runner/decode_cuda_graph_runner.py | 115 +-- .../srt/model_executor/runner/eager_runner.py | 17 +- .../runner/prefill_cuda_graph_runner.py | 10 +- .../runner/row_bucket_graph_runner.py | 25 +- python/sglang/srt/models/bailing_moe.py | 5 - python/sglang/srt/models/bailing_moe_v3.py | 5 - python/sglang/srt/models/deepseek_v2.py | 59 +- .../srt/models/deepseek_v4_replay_tail.py | 27 +- .../srt/models/dots3_common/modeling.py | 7 +- python/sglang/srt/models/exaone_moe.py | 5 - python/sglang/srt/models/glm4_moe.py | 5 - python/sglang/srt/models/glm4_moe_lite.py | 5 - python/sglang/srt/models/llada2.py | 5 - python/sglang/srt/models/longcat_flash.py | 10 + python/sglang/srt/models/mimo_v2.py | 14 - python/sglang/srt/models/minimax_m2.py | 5 - python/sglang/srt/models/minimax_m3.py | 5 - python/sglang/srt/models/nemotron_h.py | 38 +- python/sglang/srt/models/qwen2_moe.py | 19 - python/sglang/srt/models/qwen3_moe.py | 29 +- python/sglang/srt/models/sdar_moe.py | 5 - python/sglang/srt/models/step3p5.py | 5 - python/sglang/srt/models/xllm.py | 9 +- .../sglang/srt/multimodal/dsv41/vl_routing.py | 79 +- .../eagle_draft_cuda_graph_runner.py | 5 +- .../eagle_draft_extend_cuda_graph_runner.py | 3 +- .../sglang/srt/speculative/eagle_worker_v2.py | 6 +- .../frozen_kv_mtp_cuda_graph_runner.py | 5 +- .../speculative/frozen_kv_mtp_worker_v2.py | 10 +- ...er_eagle_draft_extend_cuda_graph_runner.py | 13 +- .../sglang/srt/speculative/uno_worker_v2.py | 5 +- .../moe/test_cutlass_fp8_inactive_routes.py | 274 +++++++ .../moe/test_cutlass_w4a8_inactive_routes.py | 185 +++++ .../ops/moe/test_humming_fp8_dispatch.py | 104 ++- .../kernels/ops/moe/test_moe_wna16_marlin.py | 452 ++++++++++- .../moe/test_shared_expert_routing_extra.py | 21 +- .../registered/moe/test_triton_moe_padding.py | 309 ++++++++ .../layers/moe/test_flashinfer_dispatcher.py | 15 +- .../moe/test_fused_shared_expert_scaling.py | 14 +- .../moe/test_mega_moe_decode_capacity.py | 18 +- .../layers/moe/test_mega_moe_deepgemm_api.py | 107 +-- .../layers/moe/test_topk_routing_inputs.py | 724 ++++++++++++++++++ .../quantization/test_gguf_inactive_routes.py | 342 +++++++++ .../test_mxfp4_sm100_trtllm_gen.py | 2 +- ...st_experimental_sgl_marlin_runtime_unit.py | 28 +- .../runner/test_graph_routing_context.py | 350 +++++++++ .../test_num_token_non_padded_localization.py | 194 ++++- .../models/test_draft_entry_hook_parity.py | 5 + .../test_longcat_flash_router_hpc_gemm.py | 37 +- 83 files changed, 3915 insertions(+), 641 deletions(-) create mode 100644 python/sglang/kernels/ops/moe/shuffle_weighted_sum.py create mode 100644 test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py create mode 100644 test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py create mode 100644 test/registered/moe/test_triton_moe_padding.py create mode 100644 test/registered/unit/layers/moe/test_topk_routing_inputs.py create mode 100644 test/registered/unit/layers/quantization/test_gguf_inactive_routes.py create mode 100644 test/registered/unit/model_executor/runner/test_graph_routing_context.py diff --git a/.github/workflows/_pr-test-simulator-gpu.yml b/.github/workflows/_pr-test-simulator-gpu.yml index 6d247dfe66..cef321006f 100644 --- a/.github/workflows/_pr-test-simulator-gpu.yml +++ b/.github/workflows/_pr-test-simulator-gpu.yml @@ -43,7 +43,7 @@ jobs: with: path: python/sglang/kernels/aot/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda* + pattern: wheel-python3.11-cuda* - uses: ./.github/actions/download-rust-ext with: diff --git a/python/sglang/kernels/aot/csrc/common_extension.cc b/python/sglang/kernels/aot/csrc/common_extension.cc index 1b8d68cc17..49816a619f 100644 --- a/python/sglang/kernels/aot/csrc/common_extension.cc +++ b/python/sglang/kernels/aot/csrc/common_extension.cc @@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec); m.def("ggml_moe_get_block_size(int type) -> int"); - m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size); + m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size); /* * From csrc/mamba diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh b/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh index 91e434fa60..6b48201c5a 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh +++ b/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh @@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q( const int nrows_y, const int nrows_dst, const int top_k) { + if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return; + const int blocks_per_row_x = ncols_x / qk; const int blocks_per_col_y = nrows_y / QK8_1; const int blocks_per_warp = WARP_SIZE_GGUF / qi; @@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q( } const int exp_idx = expert_ids[blockIdx.y]; - if (exp_idx > 255 || exp_idx < 0) return; - if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return; + if (exp_idx < 0) { +#pragma unroll + for (int j = 0; j < mmq_x; j += nwarps) { + const int col_dst = token_offs[j / nwarps]; + if (col_dst >= ncols_dst) continue; +#pragma unroll + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) { + const auto row_dst = row_dst_0 + threadIdx.x + i; + if (row_dst < nrows_dst) { + dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f); + } + } + } + return; + } + if (exp_idx > 255) return; const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride); const block_q8_1* y = (const block_q8_1*)(vy); diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh b/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh index 8cef9e080a..174736144b 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh +++ b/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh @@ -20,6 +20,12 @@ static __global__ void moe_vec_q( if (row >= nrows) { return; } + if (expert < 0) { + if (threadIdx.x == 0) { + dst[blockIdx.z * nrows + row] = scalar_t(0.0f); + } + return; + } const int blocks_per_row = ncols / qk; const int blocks_per_warp = vdr * WARP_SIZE / qi; diff --git a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h index e5f1224b6d..010ff58ec3 100644 --- a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h +++ b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h @@ -383,14 +383,17 @@ __global__ void Marlin( int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; int parallel = num_tokens_past_padded / moe_block_size; - int num_valid_blocks = parallel; - if constexpr (kIsEP) { - for (int i = 0; i < parallel; i++) { - if (expert_ids_ptr[i] == -1) num_valid_blocks--; - } + // Every warp counts the same inactive blocks without a serial per-thread scan. + int num_invalid_blocks = 0; + for (int i = threadIdx.x % 32; i < parallel; i += 32) { + num_invalid_blocks += expert_ids_ptr[i] == -1; } - int num_invalid_blocks = parallel - num_valid_blocks; - parallel = num_valid_blocks; +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset); + } + parallel -= num_invalid_blocks; + if (parallel == 0) return; int k_tiles = prob_k / 16 / thread_k_blocks; int n_tiles = prob_n / 16 / thread_n_blocks; @@ -518,23 +521,18 @@ __global__ void Marlin( if (par_id >= parallel) return; old_expert_id = expert_id; - if constexpr (kIsEP) { - if (num_invalid_blocks > 0) { - int skip_count = block_id == -1 ? par_id : 0; - block_id++; - for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { - expert_id = expert_ids_ptr[i]; - if (expert_id != -1) { - if (skip_count == 0) { - block_id = i; - break; - }; - skip_count--; + if (num_invalid_blocks > 0) { + int skip_count = block_id == -1 ? par_id : 0; + block_id++; + for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { + expert_id = expert_ids_ptr[i]; + if (expert_id != -1) { + if (skip_count == 0) { + block_id = i; + break; }; - } - } else { - block_id = par_id; - expert_id = expert_ids_ptr[block_id]; + skip_count--; + }; } } else { block_id = par_id; diff --git a/python/sglang/kernels/jit/csrc/moe/moe_permute_prepare.cu b/python/sglang/kernels/jit/csrc/moe/moe_permute_prepare.cu index 6e09a59985..7a65469ef3 100644 --- a/python/sglang/kernels/jit/csrc/moe/moe_permute_prepare.cu +++ b/python/sglang/kernels/jit/csrc/moe/moe_permute_prepare.cu @@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel( bool is_ep) { int tid = blockIdx.x * blockDim.x + threadIdx.x; int stride = gridDim.x * blockDim.x; - int32_t neg_count = 0; - if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0); + // Padding and nonlocal routes are both inactive, independently of EP. + int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0); for (int e = tid; e <= num_experts; e += stride) { int32_t offset; diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index a825e84ff2..b92ddfcb14 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -60,6 +60,16 @@ register_kernel( ) +register_kernel( + KernelSpec( + op="moe.shuffle_weighted_sum", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum", + capabilities=CUDA_ONLY, + ) +) + + def moe_align_block_size( topk_ids: torch.Tensor, num_experts: int, diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index 4ed917db39..2b95264024 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq( use_int4_w4a16: tl.constexpr, use_int8_w8a16: tl.constexpr, even_Ks: tl.constexpr, - filter_expert: tl.constexpr, ): """ Implements the fused computation for a Mixture of Experts (MOE) using @@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq( token_mask = offs_token < num_valid_tokens off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64) - if filter_expert and off_experts == -1: - # ----------------------------------------------------------- - # Write back zeros to the output when the expert is not - # in the current expert parallel rank. + if off_experts == -1: + # Padding and nonlocal routes both use -1, including without EP. write_zeros_to_output( c_ptr, stride_cm, @@ -377,12 +374,10 @@ def fused_moe_kernel( per_channel_quant: tl.constexpr, even_Ks: tl.constexpr, c_sorted: tl.constexpr, - filter_expert: tl.constexpr, swap_ab: tl.constexpr, FUSE_ADD_TO_OUTPUT: tl.constexpr, MASK_OUTPUT: tl.constexpr, FUSE_SUM_ALL_REDUCE: tl.constexpr, - LORA_PRESERVE_BASE: tl.constexpr, ROUTER_TOPK: tl.constexpr, FUSE_SWIGLU: tl.constexpr = False, USE_GDC: tl.constexpr = False, @@ -450,16 +445,10 @@ def fused_moe_kernel( off_experts_i32 = tl.load(expert_ids_ptr + pid_m) off_experts = off_experts_i32.to(tl.int64) - if filter_expert and off_experts == -1: - if FUSE_SWIGLU: - # C is the half-width post-activation buffer here. Rows owned by a - # filtered expert are never read (the down-GEMM CTA for this block - # early-exits before loading A), and an N-wide zero store would run - # past the row end into a neighboring token's data. - return - if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE): - # Write zeros only when this kernel owns the full output; the experimental LoRA - # add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel. + if off_experts == -1: + # The down GEMM never reads inactive sorted or half-width intermediates. + # Add/reduce outputs already hold their initial value and must stay intact. + if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE): write_zeros_to_output( c_ptr, stride_cm, @@ -822,6 +811,7 @@ def invoke_fused_moe_kernel( lora_preserve_base: bool = False, fuse_swiglu: bool = False, ) -> None: + """Inactive expert IDs are always skipped, including with filter_expert=False.""" assert topk_weights.stride(1) == 1 assert sorted_token_ids.stride(0) == 1 @@ -974,7 +964,6 @@ def invoke_fused_moe_kernel( use_int4_w4a16=use_int4_w4a16, use_int8_w8a16=use_int8_w8a16, even_Ks=even_Ks, - filter_expert=filter_expert, **config, ) @@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel( per_channel_quant=per_channel_quant, even_Ks=even_Ks, c_sorted=c_sorted, - filter_expert=filter_expert, swap_ab=swap_ab, FUSE_ADD_TO_OUTPUT=fuse_add_to_output, MASK_OUTPUT=mask_output, - LORA_PRESERVE_BASE=lora_preserve_base, FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce, ROUTER_TOPK=router_topk, FUSE_SWIGLU=fuse_swiglu, diff --git a/python/sglang/kernels/ops/moe/moe_fused_mul_sum.py b/python/sglang/kernels/ops/moe/moe_fused_mul_sum.py index 94f57ffbf5..df7ecfd147 100644 --- a/python/sglang/kernels/ops/moe/moe_fused_mul_sum.py +++ b/python/sglang/kernels/ops/moe/moe_fused_mul_sum.py @@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel( num_tokens, stride_m, has_expert_map: tl.constexpr, - is_ep: tl.constexpr, + has_topk_ids: tl.constexpr, top_k: tl.constexpr, size: tl.constexpr, routed_scaling_factor: tl.constexpr, @@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel( if routed_scaling_factor != 1.0: b_val = b_val * routed_scaling_factor if has_expert_map: - id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0) - expert_mask = tl.load(expert_map_ptr + id_val) >= 0 + id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1) + mapped_id = tl.load( + expert_map_ptr + tl.maximum(id_val, 0), + mask=m_mask & (id_val >= 0), + other=-1, + ) + expert_mask = mapped_id >= 0 + b_val = tl.where(expert_mask, b_val, 0.0) a_vec = tl.load( a_base + n * size, mask=mask & expert_mask[:, None], other=0.0, ).to(tl.float32) - elif is_ep: + elif has_topk_ids: id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0) expert_mask = id_val >= 0 + b_val = tl.where(expert_mask, b_val, 0.0) a_vec = tl.load( a_base + n * size, mask=mask & expert_mask[:, None], @@ -164,8 +171,8 @@ def moe_fused_mul_sum( Shape: (num_tokens, top_k). outputs: Optional pre-allocated output tensor. Shape: (num_tokens, hidden_size). - topk_ids: Optional indices of the top-k experts. Used when - `expert_map` is provided. Shape: (num_tokens, top_k). + topk_ids: Optional indices of the top-k experts. Negative ids always + skip the corresponding input load. Shape: (num_tokens, top_k). expert_map: Optional mapping for Expert Parallelism. A value < 0 indicates an invalid token/expert pair that will be skipped. @@ -180,6 +187,7 @@ def moe_fused_mul_sum( assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16) assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16) + # Keep is_ep as a compatible host argument; route validity follows ids. num_tokens, top_k, size = inputs.shape output_shape = (num_tokens, size) if outputs is None: @@ -205,7 +213,7 @@ def moe_fused_mul_sum( num_tokens, top_k * size, expert_map is not None, - is_ep, + topk_ids is not None, top_k, size, 1.0 if routed_scaling_factor is None else routed_scaling_factor, diff --git a/python/sglang/kernels/ops/moe/moe_wna16_marlin.py b/python/sglang/kernels/ops/moe/moe_wna16_marlin.py index 1defc8a92d..e5d9d6a6f6 100644 --- a/python/sglang/kernels/ops/moe/moe_wna16_marlin.py +++ b/python/sglang/kernels/ops/moe/moe_wna16_marlin.py @@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm( use_fp32_reduce: bool = False, is_zp_float: bool = False, ) -> torch.Tensor: + """Project routes, preserving inactive rows in a caller-provided output. + + Inactive rows are zero when this function allocates the output. + """ device = a.device # Allocate output if not provided if c_or_none is not None: c = c_or_none else: - c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device) + c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device) # Early return for zero-size M if size_m == 0: diff --git a/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py b/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py new file mode 100644 index 0000000000..4a1b51235a --- /dev/null +++ b/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py @@ -0,0 +1,74 @@ +"""Reduce routed rows while skipping inactive permutation entries.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"]) +def _shuffle_weighted_sum_kernel( + inputs, + output, + permutation, + weights, + width: tl.constexpr, + rows, + topk: tl.constexpr, + input_stride: tl.constexpr, + output_stride: tl.constexpr, + weight_stride0, + weight_stride1, + BLOCK: tl.constexpr, +): + token = tl.program_id(0) + columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + total = tl.full((BLOCK,), 0, tl.float32) + for slot in range(topk): + row = tl.load(permutation + token * topk + slot).to(tl.int64) + active = (row >= 0) & (row < rows) + values = tl.load( + inputs + row * input_stride + columns, + mask=active & (columns < width), + other=0, + ).to(tl.float32) + factor = tl.load( + weights + token * weight_stride0 + slot * weight_stride1, + mask=active, + other=0, + ).to(tl.float32) + total += values * factor + tl.store( + output + token.to(tl.int64) * output_stride + columns, + total, + mask=columns < width, + ) + + +def shuffle_weighted_sum( + inputs: torch.Tensor, + output: torch.Tensor, + permutation: torch.Tensor, + weights: torch.Tensor, +) -> None: + """Sum mapped expert outputs; inactive maps read neither data nor weights.""" + assert inputs.ndim == output.ndim == weights.ndim == 2 + assert inputs.shape[1] == output.shape[1] + assert inputs.stride(1) == output.stride(1) == 1 + assert weights.shape[0] == output.shape[0] + assert permutation.is_contiguous() and permutation.numel() == weights.numel() + if output.numel() == 0: + return + _shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))]( + inputs, + output, + permutation, + weights, + output.shape[1], + inputs.shape[0], + weights.shape[1], + inputs.stride(0), + output.stride(0), + weights.stride(0), + weights.stride(1), + BLOCK=256, + ) diff --git a/python/sglang/srt/batch_overlap/operations.py b/python/sglang/srt/batch_overlap/operations.py index 5e8829277c..76243a7882 100644 --- a/python/sglang/srt/batch_overlap/operations.py +++ b/python/sglang/srt/batch_overlap/operations.py @@ -125,6 +125,10 @@ class _StageExecutor: # handling DP attention forward_batch: ForwardBatch = inputs["forward_batch"] + self._child_ctx = replace( + child_ctx if child_ctx is not None else get_forward_context(), + moe_num_token_non_padded=forward_batch.moe_num_token_non_padded, + ) self._global_dp_buffer_len = forward_batch.global_dp_buffer_len self._local_dp_buffer_len = forward_batch.tbo_padded_len self._global_num_tokens = forward_batch.global_num_tokens_cpu diff --git a/python/sglang/srt/eplb/expert_location_dispatch.py b/python/sglang/srt/eplb/expert_location_dispatch.py index 686ec6ea8d..beca79ab6e 100644 --- a/python/sglang/srt/eplb/expert_location_dispatch.py +++ b/python/sglang/srt/eplb/expert_location_dispatch.py @@ -39,13 +39,16 @@ class ExpertLocationDispatchInfo: rank_invariant: bool = False @classmethod - def init_new(cls, layer_id: int): + def init_new(cls, layer_id: int, *, is_draft: bool = False): + if is_draft: + return None ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm - expert_location_metadata = get_global_expert_location_metadata() - assert expert_location_metadata is not None - if ep_dispatch_algorithm is None: return None + expert_location_metadata = get_global_expert_location_metadata() + assert expert_location_metadata is not None, ( + "Expert dispatch requires placement metadata" + ) return cls( ep_dispatch_algorithm=ep_dispatch_algorithm, diff --git a/python/sglang/srt/hardware_backend/npu/moe/topk.py b/python/sglang/srt/hardware_backend/npu/moe/topk.py index 117096bb50..3ced01ee47 100644 --- a/python/sglang/srt/hardware_backend/npu/moe/topk.py +++ b/python/sglang/srt/hardware_backend/npu/moe/topk.py @@ -1,12 +1,12 @@ from typing import TYPE_CHECKING, Optional import torch -from sgl_kernel_npu.norm.l1_norm import l1_norm from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical from sglang.srt.layers.moe.topk import ( StandardTopKOutput, + _zero_topk_weights_padded_region, capture_routed_experts_if_allowed, select_experts, ) @@ -59,6 +59,8 @@ def fused_topk_npu( ) if renormalize: + from sgl_kernel_npu.norm.l1_norm import l1_norm + topk_weights = l1_norm( topk_weights if topk_config.num_shared_topk_slots == 0 @@ -109,9 +111,44 @@ def fused_topk_npu( expert_location_dispatch_info=expert_location_dispatch_info, ) - if expert_location_dispatch_info is not None: - topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) - get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids) - capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids) + return _finalize_topk_npu( + topk_ids=topk_ids, + topk_weights=topk_weights, + router_logits=router_logits, + topk_config=topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + layer_id=layer_id, + ) + +def _finalize_topk_npu( + *, + topk_ids, + topk_weights, + router_logits, + topk_config, + num_token_non_padded, + expert_location_dispatch_info, + layer_id, +): + num_shared = topk_config.num_shared_topk_slots + routed_ids = topk_ids[:, :-num_shared] if num_shared else topk_ids + capture_routed_experts_if_allowed(topk_config, layer_id, routed_ids) + routed_ids = topk_ids_logical_to_physical(routed_ids, expert_location_dispatch_info) + topk_ids = ( + torch.cat([routed_ids, topk_ids[:, -num_shared:]], dim=-1) + if num_shared + else routed_ids + ) + # NPU dispatch retains valid IDs; duplicating a padded row's routes onto + # expert zero can exceed per-expert capacity in fused dispatch backends. + _zero_topk_weights_padded_region(topk_weights, num_token_non_padded) + recorder_ids = routed_ids + if num_token_non_padded is not None: + rows = torch.arange(topk_ids.shape[0], device=topk_ids.device) + recorder_ids = routed_ids.masked_fill( + (rows >= num_token_non_padded)[:, None], -1 + ) + get_global_expert_distribution_recorder().on_select_experts(topk_ids=recorder_ids) return StandardTopKOutput(topk_weights, topk_ids, router_logits) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 35054b7670..d25aaf1ade 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -1209,10 +1209,6 @@ class AiterAttnBackend(AttentionBackend): bs, forward_batch.seq_lens, forward_batch.req_pool_indices, max_kv_len ) - # Build cu_seqlens_q from this batch's extend lengths. The standard - # prefill metadata path leaves self.qo_indptr unset (qo_indptr=None in - # ForwardMetadata), so relying on it corrupts multi-sequence batches - # (only bs=1 happens to work). cu_seqlens_q = torch.zeros(bs + 1, dtype=torch.int32, device=q.device) cu_seqlens_q[1:] = torch.cumsum( forward_batch.extend_seq_lens.to(torch.int32), dim=0 @@ -2598,7 +2594,7 @@ class AiterAttnBackend(AttentionBackend): self.forward_metadata = ForwardMetadata( self.indices_updater_prefill.kv_indptr, self.indices_updater_prefill.kv_indices, - None, + self.indices_updater_prefill.qo_indptr[: bs + 1], None, max(forward_batch.extend_seq_lens_cpu), forward_batch.seq_lens_cpu.max().item(), @@ -4423,7 +4419,7 @@ class AiterAttnBackend(AttentionBackend): self.k_scale.reshape(1), # Q is cast at unit scale. k_descale.reshape(1), v_descale.reshape(1), - self.qo_indptr[:bs0], + self.forward_metadata.qo_indptr[:bs0], cu_k, self.forward_metadata.max_q_len, int(self.forward_metadata.max_kv_len), @@ -4499,7 +4495,7 @@ class AiterAttnBackend(AttentionBackend): fp8_q_descale.reshape(1), k_descale.reshape(1), v_descale.reshape(1), - self.qo_indptr[:bs0], + self.forward_metadata.qo_indptr[:bs0], cu_k, self.forward_metadata.max_q_len, int(self.forward_metadata.max_kv_len), @@ -4538,8 +4534,8 @@ class AiterAttnBackend(AttentionBackend): fp8_q_descale.reshape(1), k_descale.reshape(1), v_descale.reshape(1), - self.qo_indptr[:bs0], - self.qo_indptr[:bs0], + self.forward_metadata.qo_indptr[:bs0], + self.forward_metadata.qo_indptr[:bs0], self.forward_metadata.max_q_len, self.forward_metadata.max_q_len, softmax_scale=layer.scaling, @@ -4628,7 +4624,7 @@ class AiterAttnBackend(AttentionBackend): q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_cache, v_cache, - self.qo_indptr[:bs0], + self.forward_metadata.qo_indptr[:bs0], kv_indptr_arg, page_table, self.forward_metadata.max_q_len, diff --git a/python/sglang/srt/layers/attention/aiter_utils.py b/python/sglang/srt/layers/attention/aiter_utils.py index 4429f83614..4d1966b2fb 100644 --- a/python/sglang/srt/layers/attention/aiter_utils.py +++ b/python/sglang/srt/layers/attention/aiter_utils.py @@ -94,13 +94,13 @@ def forward_extend_vectorized_5d( kv_indices_lin = torch.arange( total_tokens, dtype=torch.int32, device=k_lin.device ) - kv_indptr_lin = backend.qo_indptr[:bs0] + kv_indptr_lin = backend.forward_metadata.qo_indptr[:bs0] max_q = int(backend.forward_metadata.max_q_len) o = mha_batch_prefill_func( q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_lin, v_lin, - backend.qo_indptr[:bs0], + backend.forward_metadata.qo_indptr[:bs0], kv_indptr_lin, kv_indices_lin, max_q, @@ -187,7 +187,7 @@ def forward_extend_vectorized_5d( q_local.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_lin, v_lin, - backend.qo_indptr[:bs0], + backend.forward_metadata.qo_indptr[:bs0], kv_indptr_lin, kv_indices_lin, max_q, diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py index b9c9e37634..b2b8ffa3cd 100755 --- a/python/sglang/srt/layers/moe/cutlass_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -10,7 +10,6 @@ from sglang.srt.utils import is_cuda _is_cuda = is_cuda() if _is_cuda: from sgl_kernel import ( - apply_shuffle_mul_sum, es_fp8_blockwise_scaled_grouped_mm, es_sm100_mxfp8_blockscaled_grouped_mm, es_sm100_mxfp8_blockscaled_grouped_quant, @@ -23,6 +22,7 @@ if _is_cuda: from sglang.kernels.ops.moe.shuffle_rows_with_scales import ( shuffle_rows_with_scales, ) + from sglang.kernels.ops.moe.shuffle_weighted_sum import shuffle_weighted_sum def cutlass_fused_experts_fp8( @@ -142,8 +142,9 @@ def cutlass_fused_experts_fp8( topk = topk_ids.size(1) device = a.device - a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) - c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + # prepare_moe_input writes only live routes; gathers still span the tail. + a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device) + c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device) if use_mxfp8: assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs" @@ -345,5 +346,5 @@ def cutlass_fused_experts_fp8( if output is None: output = torch.empty((m, k), device=device, dtype=out_dtype) - apply_shuffle_mul_sum(c2, output, c_map, topk_weights.to(out_dtype)) + shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype)) return output diff --git a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py index 0efbd78781..f6b3c08326 100644 --- a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py @@ -5,7 +5,6 @@ from typing import Optional import torch -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_cuda, is_cuda_alike _is_cuda = is_cuda() @@ -128,8 +127,8 @@ def cutlass_w4a8_moe( assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1" device = a.device - if get_parallel().moe_ep_size > 1: - topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids) + # Both graph padding and nonlocal routes use -1, including without EP. + topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids) src2dst = cutlass_w4_run_moe_ep_preproess( topk_ids, @@ -144,7 +143,11 @@ def cutlass_w4a8_moe( # TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe if a1_scale is None: a1_scale = torch.zeros(1, dtype=torch.float32, device=device) - per_tensor_absmax_fp8(a, a1_scale) + # Inactive rows must not change the quantization of live expert inputs. + inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True) + scale_input = a.masked_fill(inactive_rows, 0.0) + per_tensor_absmax_fp8(scale_input, a1_scale) + a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max) pre_reorder_for_cutlass_moe( a, diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py index a3617a5721..53ca979a39 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py @@ -305,15 +305,14 @@ def fused_marlin_moe( device=hidden_states.device, dtype=hidden_states.dtype, ) - # Marlin skips masked expert rows, so their shared cache must start at zero. - intermediate_cache13 = torch.zeros( - (M * topk_ids.shape[1] * max(gemm1_n, K),), + intermediate_cache13 = torch.empty( + (M * topk * max(gemm1_n, K),), device=hidden_states.device, dtype=hidden_states.dtype, ) intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * gemm1_n] intermediate_cache1 = intermediate_cache1.view(-1, gemm1_n) - intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K] + intermediate_cache3 = intermediate_cache13[: M * topk * K] intermediate_cache3 = intermediate_cache3.view(-1, K) use_atomic_add = ( @@ -381,9 +380,9 @@ def fused_marlin_moe( else: raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}") - if expert_map is not None: - intermediate_cache3.zero_() - + # Gate-up is dead after activation, so its storage can hold the down output. + # Skipped down routes must start at zero before the top-k reduction. + intermediate_cache3.zero_() intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, intermediate_cache3, diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 3a3e527690..6371452dff 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -74,6 +74,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight from sglang.srt.runtime_context import ( get_exec, + get_flags, get_global_dwdp_manager, get_parallel, get_server_args, @@ -367,7 +368,8 @@ class FusedMoE(torch.nn.Module): self.params_dtype = params_dtype self.layer_name = prefix self.layer_id = layer_id - self.uses_expert_location = uses_expert_location + self.is_draft = get_flags().moe.in_speculative_scope + self.uses_expert_location = uses_expert_location and not self.is_draft self.top_k = top_k self.hidden_size = hidden_size self.num_experts = num_experts @@ -1559,6 +1561,14 @@ class FusedMoE(torch.nn.Module): return forward_fuseep(self, hidden_states, topk_output) if is_in_tc_piecewise_cuda_graph(): + if ( + TopKOutputChecker.format_is_bypassed(topk_output) + and topk_output.expert_location_dispatch_info is not None + ): + # The opaque logits-only op cannot carry placement metadata. + return self.forward_impl( + hidden_states, topk_output, pre_quant_input=pre_quant_input + ) if TopKOutputChecker.format_is_standard(topk_output): return moe_forward_piecewise_cuda_graph_impl( hidden_states, @@ -1872,7 +1882,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl( ) -> torch.Tensor: topk_output = BypassedTopKOutput( hidden_states=hidden_states, - router_logits=router_logits, + raw_router_logits=router_logits, topk_config=TopKConfig( top_k=top_k, topk_group=topk_group, @@ -1880,6 +1890,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl( correction_bias=correction_bias, renormalize=renormalize, allow_routed_experts_capture=allow_routed_experts_capture, + layer_id=layer_id, ), ) forward_context = get_tc_piecewise_forward_context() diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 205ae1ab9f..10ea97f40e 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -21,11 +21,10 @@ from sglang.srt.layers.moe.topk import ( _mask_topk_ids_padded_region, _zero_topk_weights_padded_region, remap_topk_for_per_rank_shared_slots, + resolve_routing_inputs, ) -from sglang.srt.layers.moe.utils import ( - has_per_rank_fused_shared_slots, -) -from sglang.srt.runtime_context import get_model +from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots +from sglang.srt.runtime_context import get_flags, get_model from sglang.srt.utils import is_hip, is_npu, is_xpu logger = logging.getLogger(__name__) @@ -46,9 +45,13 @@ class HashTopK(nn.Module): routed_scaling_factor=1.5, apply_routed_scaling_factor_on_output=False, layer_id: Optional[int] = None, + is_draft: Optional[bool] = None, ): super().__init__() self.layer_id = layer_id + self.is_draft = ( + get_flags().moe.in_speculative_scope if is_draft is None else is_draft + ) self.shared_experts = get_model().shared_experts self.enable_waterfill = self.shared_experts.use_waterfill @@ -75,6 +78,9 @@ class HashTopK(nn.Module): "with fused shared experts; pass --disable-shared-experts-fusion." ) + def routing_inputs(self): + return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft) + @property def num_shared_topk_slots(self) -> int: return self.shared_experts.num_shared_topk_slots @@ -100,7 +106,8 @@ class HashTopK(nn.Module): self, device: torch.device, *, layer_id: Optional[int] = None ): topk = self.topk - self.num_shared_topk_slots - if layer_id is not None: + if not self.is_draft: + layer_id = self.layer_id from sglang.srt.eplb.lplb_solver import get_global_lplb_solver lplb_solver = get_global_lplb_solver(layer_id) @@ -226,6 +233,12 @@ class HashTopK(nn.Module): num_token_non_padded: Optional[torch.Tensor] = None, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, ): + num_token_non_padded, expert_location_dispatch_info = resolve_routing_inputs( + layer_id=self.layer_id, + is_draft=self.is_draft, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) assert input_ids.shape[0] == hidden_states.shape[0] == router_logits.shape[0], ( f"{input_ids.shape=} {hidden_states.shape=} {router_logits.shape=}" ) @@ -252,6 +265,9 @@ class HashTopK(nn.Module): topk_weights = topk_weights * self.routed_scaling_factor num_shared_topk_slots = self.num_shared_topk_slots + routed_ids = ( + topk_ids[:, :-num_shared_topk_slots] if num_shared_topk_slots else topk_ids + ) log2phy_prob = None if ( expert_location_dispatch_info is not None @@ -263,18 +279,13 @@ class HashTopK(nn.Module): lplb_solver = get_global_lplb_solver(self.layer_id) if lplb_solver is not None: - log2phy_prob = lplb_solver.solve(topk_ids) + log2phy_prob = lplb_solver.solve(routed_ids) - recorder_topk_ids = None + routed_cols = topk_ids_logical_to_physical( + routed_ids, expert_location_dispatch_info, log2phy_prob + ) + recorder_topk_ids = routed_cols if num_shared_topk_slots else None if has_per_rank_fused_shared_slots(num_shared_topk_slots): - # The trailing columns are placeholders; the remap appends the real ones. - routed_cols = topk_ids_logical_to_physical( - topk_ids[:, :-num_shared_topk_slots], - expert_location_dispatch_info, - log2phy_prob, - ) - recorder_topk_ids = routed_cols - num_physical_routed_experts = ( expert_location_dispatch_info.num_physical_experts if expert_location_dispatch_info is not None @@ -292,11 +303,13 @@ class HashTopK(nn.Module): apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output, ), ) - else: - topk_ids = topk_ids_logical_to_physical( - topk_ids, expert_location_dispatch_info, log2phy_prob + elif num_shared_topk_slots: + topk_ids = torch.cat( + [routed_cols, topk_ids[:, -num_shared_topk_slots:]], dim=-1 ) - if is_hip(): + else: + topk_ids = routed_cols + if _is_hip or _is_npu: _zero_topk_weights_padded_region(topk_weights, num_token_non_padded) else: _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) @@ -304,6 +317,11 @@ class HashTopK(nn.Module): _mask_topk_ids_padded_region(recorder_topk_ids, num_token_non_padded) if recorder_topk_ids is None: recorder_topk_ids = topk_ids + if _is_npu and num_token_non_padded is not None: + rows = torch.arange(topk_ids.shape[0], device=topk_ids.device) + recorder_topk_ids = recorder_topk_ids.masked_fill( + (rows >= num_token_non_padded)[:, None], -1 + ) get_global_expert_distribution_recorder().on_select_experts( topk_ids=recorder_topk_ids ) @@ -311,7 +329,7 @@ class HashTopK(nn.Module): topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits ) topk_output = self._apply_waterfill(topk_output, hidden_states.shape[0]) - if is_hip(): + if _is_hip or _is_npu: _zero_topk_weights_padded_region( topk_output.topk_weights, num_token_non_padded ) diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index e97d0f0895..7bf90bf095 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -23,7 +23,6 @@ import torch from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch from sglang.srt.environ import envs -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.dp_attention import get_dp_global_num_tokens from sglang.srt.layers.moe.mega_moe_sm90 import ( @@ -263,16 +262,6 @@ def _run_mega_routed( if num_tokens > 0: router_logits = moe.gate(hidden_states, forward_batch=forward_batch) topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {} - num_token_non_padded = ( - forward_batch.moe_num_token_non_padded() - if forward_batch is not None - else None - ) - dispatch_info = ( - ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id) - if moe.experts.uses_expert_location - else None - ) if ( not moe.is_hash and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None @@ -281,15 +270,11 @@ def _run_mega_routed( moe, router_logits, input_ids_global, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=dispatch_info, ) else: topk_output = moe.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=dispatch_info, **topk_kwargs, ) topk_ids = topk_output.topk_ids diff --git a/python/sglang/srt/layers/moe/mega_moe_flydsl.py b/python/sglang/srt/layers/moe/mega_moe_flydsl.py index aca30fd28a..c858e1334d 100644 --- a/python/sglang/srt/layers/moe/mega_moe_flydsl.py +++ b/python/sglang/srt/layers/moe/mega_moe_flydsl.py @@ -14,7 +14,6 @@ import torch from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows from sglang.srt.environ import envs -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.dp_attention import ( get_dp_global_num_tokens, @@ -307,25 +306,16 @@ def _run_mega_routed( topk_output = moe.topk( hidden_states, router_logits, - num_token_non_padded=( - forward_batch.num_token_non_padded - if forward_batch is not None - else None - ), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id) - if moe.experts.uses_expert_location - else None - ), **topk_kwargs, ) x_in = hidden_states topk_ids = topk_output.topk_ids.to(torch.int32) topk_weights = topk_output.topk_weights.to(torch.float32) - if forward_batch is not None and forward_batch.num_token_non_padded is not None: + num_token_non_padded, _ = moe.topk.routing_inputs() + if num_token_non_padded is not None: _fill_padded_rows( topk_ids, - forward_batch.num_token_non_padded, + num_token_non_padded, experts, ) else: diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 5a3aa48ef1..3c2b9ca785 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -41,6 +41,7 @@ if TYPE_CHECKING: from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.runtime_context import ( get_exec, + get_flags, get_forward, get_lora, get_model, @@ -110,6 +111,10 @@ from sglang.srt.layers.moe.utils import ( RoutingMethodType, has_per_rank_fused_shared_slots, ) +from sglang.srt.model_executor.forward_context import ( + get_forward_context, + has_forward_context, +) from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import ( cpu_has_amx_support, @@ -279,6 +284,8 @@ class TopKConfig: # Draft-side MoE blocks set this False so they never write the target's # process-global routed-experts capture buffer. allow_routed_experts_capture: bool = True + # Lazy materialization still needs the producer's solver and capture slot. + layer_id: Optional[int] = None _correction_bias_dtype_cache: Optional[torch.Tensor] = field( default=None, init=False, repr=False, compare=False ) @@ -393,10 +400,10 @@ class TritonKernelTopKOutput(NamedTuple): class BypassedTopKOutput(NamedTuple): - """Bypassed top-k output format.""" + """Lazy routing that can be materialized or consumed by an in-kernel router.""" hidden_states: torch.Tensor - router_logits: torch.Tensor + raw_router_logits: torch.Tensor topk_config: TopKConfig num_token_non_padded: Optional[torch.Tensor] = None expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None @@ -405,14 +412,23 @@ class BypassedTopKOutput(NamedTuple): def format(self) -> TopKOutputFormat: return TopKOutputFormat.BYPASSED + @property + def router_logits(self) -> torch.Tensor: + if self.expert_location_dispatch_info is not None: + raise NotImplementedError( + "Expert placement remapping requires materialized TopK routing; " + "this MoE kernel routes directly from logits." + ) + return self.raw_router_logits + def to_standard(self, layer_id: Optional[int] = None) -> StandardTopKOutput: """Materialize routing tensors. Used by MoE kernels that need explicit topk_ids / topk_weights rather than doing routing internally.""" return select_experts( hidden_states=self.hidden_states, - router_logits=self.router_logits, + router_logits=self.raw_router_logits, topk_config=self.topk_config, - layer_id=layer_id, + layer_id=self.topk_config.layer_id if layer_id is None else layer_id, num_token_non_padded=self.num_token_non_padded, expert_location_dispatch_info=self.expert_location_dispatch_info, ) @@ -630,6 +646,24 @@ def _can_bypass_trtllm_routing(config: TopKConfig) -> bool: # -------------------------------- TopK --------------------------------------- +def resolve_routing_inputs( + *, + layer_id: int, + is_draft: bool, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, +): + if num_token_non_padded is None and has_forward_context(): + get_bound = get_forward_context().moe_num_token_non_padded + if get_bound is not None: + num_token_non_padded = get_bound() + if is_draft: + expert_location_dispatch_info = None + elif expert_location_dispatch_info is None: + expert_location_dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id) + return num_token_non_padded, expert_location_dispatch_info + + class TopK(BaseFusedOp): """ Parameters: @@ -649,6 +683,7 @@ class TopK(BaseFusedOp): top_k: int, *, layer_id: int, + is_draft: Optional[bool] = None, use_grouped_topk: bool = False, topk_group: Optional[int] = None, num_expert_group: Optional[int] = None, @@ -674,6 +709,9 @@ class TopK(BaseFusedOp): assert num_expert_group is not None and topk_group is not None self.layer_id = layer_id + self.is_draft = ( + get_flags().moe.in_speculative_scope if is_draft is None else is_draft + ) self._runner_backend_bound = False shared_experts = get_model().shared_experts @@ -686,6 +724,7 @@ class TopK(BaseFusedOp): self.is_fp4_experts = is_fp4_experts self.topk_config = TopKConfig( top_k=top_k + num_shared_slots, + layer_id=layer_id, use_grouped_topk=use_grouped_topk, renormalize=renormalize, topk_group=topk_group, @@ -703,7 +742,21 @@ class TopK(BaseFusedOp): scoring_func=scoring_func, sqrtsoftplus_log1p=sqrtsoftplus_log1p, fused_gate_packed_ids=fused_gate_packed_ids, - allow_routed_experts_capture=allow_routed_experts_capture, + allow_routed_experts_capture=allow_routed_experts_capture + and not self.is_draft, + ) + + def routing_inputs(self): + return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft) + + def _resolve_routing_inputs( + self, num_token_non_padded, expert_location_dispatch_info + ): + return resolve_routing_inputs( + layer_id=self.layer_id, + is_draft=self.is_draft, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, ) def bind_runner_backend( @@ -778,6 +831,11 @@ class TopK(BaseFusedOp): dynamic_expert_bias: Optional[torch.Tensor] = None, router_logits_partials: Optional[torch.Tensor] = None, ) -> TopKOutput: + num_token_non_padded, expert_location_dispatch_info = ( + self._resolve_routing_inputs( + num_token_non_padded, expert_location_dispatch_info + ) + ) if router_logits_partials is not None: rocm_router_reduce_partials(router_logits_partials, router_logits) self.topk_config.torch_native = True @@ -805,6 +863,12 @@ class TopK(BaseFusedOp): """router_logits_partials (ROCm decode router): fp32 split-K partials whose fixed-order sum is the logits; router_logits is then a buffer the fused gate fills, and any other reader first reduces the partials into it.""" + explicit_dispatch_info = expert_location_dispatch_info is not None + num_token_non_padded, expert_location_dispatch_info = ( + self._resolve_routing_inputs( + num_token_non_padded, expert_location_dispatch_info + ) + ) if dynamic_expert_bias is not None: output_format = TopKOutputFormat.STANDARD elif self.topk_config.output_format is not None: @@ -814,6 +878,17 @@ class TopK(BaseFusedOp): "TopK requires the owning experts' runner backend or an explicit output format." ) + requires_remapping = expert_location_dispatch_info is not None and ( + explicit_dispatch_info + or is_expert_remapping_enabled() + or expert_location_dispatch_info.ep_dispatch_algorithm in ("fake", "lp") + ) + if output_format == TopKOutputFormat.TRITON_KERNEL and requires_remapping: + raise NotImplementedError( + "Expert placement remapping requires a MoE runner with standard TopK routing; " + f"got {output_format.name}." + ) + if ( router_logits_partials is not None and output_format != TopKOutputFormat.STANDARD @@ -833,6 +908,7 @@ class TopK(BaseFusedOp): router_logits, self.topk_config.top_k, sm_first=not self.topk_config.renormalize, + n_rows=num_token_non_padded, ) return TritonKernelTopKOutput( a_ragged_metadata, @@ -844,10 +920,12 @@ class TopK(BaseFusedOp): elif output_format == TopKOutputFormat.BYPASSED: return BypassedTopKOutput( hidden_states=hidden_states, - router_logits=router_logits, + raw_router_logits=router_logits, topk_config=self.topk_config, num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=expert_location_dispatch_info, + expert_location_dispatch_info=( + expert_location_dispatch_info if requires_remapping else None + ), ) else: self.topk_config.torch_native = False @@ -875,6 +953,11 @@ class TopK(BaseFusedOp): expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, ) -> TopKOutput: + num_token_non_padded, expert_location_dispatch_info = ( + self._resolve_routing_inputs( + num_token_non_padded, expert_location_dispatch_info + ) + ) topk_output = select_experts( hidden_states=hidden_states, layer_id=self.layer_id, @@ -895,6 +978,11 @@ class TopK(BaseFusedOp): expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, ) -> TopKOutput: + num_token_non_padded, expert_location_dispatch_info = ( + self._resolve_routing_inputs( + num_token_non_padded, expert_location_dispatch_info + ) + ) if dynamic_expert_bias is not None: self.topk_config.torch_native = False return select_experts( @@ -923,12 +1011,13 @@ class TopK(BaseFusedOp): ) -> TopKOutput: """Return an empty topk output for a rank with zero tokens this forward. - When ``layer_id`` is provided and the active dispatch algorithm is LP, - also calls ``LPLBSolver.solve(empty)`` so that this rank participates - in the EP all-reduce. Without this, an empty rank would skip the - collective and deadlock under DP-attention. + Target layers call their owning ``LPLBSolver.solve(empty)`` so this + rank participates in the EP all-reduce even without tokens. Draft + layers skip the target solver. The optional ``layer_id`` is retained + for caller compatibility; the router's owning layer determines the solver. """ - if layer_id is not None: + if not self.is_draft: + layer_id = self.layer_id # Skip the full ExpertLocationDispatchInfo allocation — we only # need the per-layer solver to participate in the EP all-reduce. from sglang.srt.eplb.lplb_solver import get_global_lplb_solver @@ -972,6 +1061,11 @@ class TopK(BaseFusedOp): expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, ) -> TopKOutput: + num_token_non_padded, expert_location_dispatch_info = ( + self._resolve_routing_inputs( + num_token_non_padded, expert_location_dispatch_info + ) + ) self.topk_config.torch_native = True # [NOTE] XPU device support for topk kernels # - support 'topk_softmax' and 'topk_sigmoid' @@ -1816,6 +1910,12 @@ def _mask_topk_ids_padded_region( ) -> None: if num_token_non_padded is None: return + if not topk_ids.is_cuda: + if _is_npu: + return + rows = torch.arange(topk_ids.shape[0], device=topk_ids.device) + topk_ids.masked_fill_((rows >= num_token_non_padded)[:, None], fill_value) + return # TODO: let the kernel support other dtypes if _is_cuda and topk_ids.dtype == torch.int32 and fill_value == -1: mask_topk_ids(topk_ids, num_token_non_padded) @@ -1834,6 +1934,10 @@ def _zero_topk_weights_padded_region( ): if num_token_non_padded is None: return + if not topk_weights.is_cuda: + rows = torch.arange(topk_weights.shape[0], device=topk_weights.device) + topk_weights.masked_fill_((rows >= num_token_non_padded)[:, None], 0.0) + return if _can_fuse_padded_region(topk_weights): _fill_padded_rows(topk_weights, num_token_non_padded, 0.0) return @@ -2420,12 +2524,27 @@ def _post_process_topk_ids( lplb_solver = get_global_lplb_solver(layer_id) if lplb_solver is not None: - log2phy_prob = lplb_solver.solve(topk_ids) + lp_routed_ids = ( + topk_ids[:, :-num_shared_topk_slots] + if router_wrote_shared_cols + else topk_ids + ) + log2phy_prob = lplb_solver.solve(lp_routed_ids) if log2phy_prob is not None: - topk_ids = topk_ids_logical_to_physical( - topk_ids, expert_location_dispatch_info, log2phy_prob - ) + if router_wrote_shared_cols: + shared_cols = topk_ids[:, -num_shared_topk_slots:] + routed_cols = topk_ids_logical_to_physical( + topk_ids[:, :-num_shared_topk_slots], + expert_location_dispatch_info, + log2phy_prob, + ) + topk_ids = torch.cat([routed_cols, shared_cols], dim=-1) + recorder_topk_ids = topk_ids[:, :-num_shared_topk_slots] + else: + topk_ids = topk_ids_logical_to_physical( + topk_ids, expert_location_dispatch_info, log2phy_prob + ) _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) elif router_wrote_shared_cols: # Shared IDs are outside EPLB's routed-expert table for both global @@ -2506,6 +2625,21 @@ def _post_process_topk_ids( # That final pass re-zeros after any shared-expert append/remap, so a # second zeroing here would be redundant (zeroing is idempotent). + else: + if num_shared_topk_slots > 0: + shared_cols = topk_ids[:, -num_shared_topk_slots:] + routed_cols = topk_ids_logical_to_physical( + topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info + ) + _mask_topk_ids_padded_region(routed_cols, num_token_non_padded) + topk_ids = torch.cat([routed_cols, shared_cols], dim=-1) + recorder_topk_ids = routed_cols + else: + topk_ids = topk_ids_logical_to_physical( + topk_ids, expert_location_dispatch_info + ) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + if recorder_topk_ids is None and not recorder_was_fused: recorder_topk_ids = ( topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids @@ -2598,6 +2732,14 @@ def _post_process_topk_ids( # supplied 1/ep_size factor the shared contribution is summed ep_size times. topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor + if _is_npu: + _zero_topk_weights_padded_region(topk_weights, num_token_non_padded) + if recorder_topk_ids is not None and num_token_non_padded is not None: + rows = torch.arange(topk_ids.shape[0], device=topk_ids.device) + recorder_topk_ids = recorder_topk_ids.masked_fill( + (rows >= num_token_non_padded)[:, None], -1 + ) + if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append: # Shared-expert append/remap can introduce non-zero weights after the # initial HIP padding mask above. Ensure padded tokens leave this helper @@ -2750,10 +2892,6 @@ def select_experts( apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, ) elif torch_native and custom_routing_function is None: - assert num_token_non_padded is None, ( - "num_token_non_padded is not yet supported in fused_topk_native" - ) - assert expert_location_dispatch_info is None fold_scale_after_routing = True topk_weights, topk_ids = fused_topk_native( hidden_states=hidden_states, diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py index cd7beaf8bb..61313d2ac9 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -387,6 +387,9 @@ class MuseSpark1xFp4Experts(nn.Module): from sglang.kernels.ops.activation.activation import silu_and_mul from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size + from sglang.srt.layers.moe.token_dispatcher.standard import ( + _map_topk_ids_to_local_experts, + ) from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace hidden_states = hidden_states.contiguous() @@ -398,7 +401,7 @@ class MuseSpark1xFp4Experts(nn.Module): local_topk_ids = topk_ids.to(torch.int32) if expert_map is not None: - local_topk_ids = expert_map[local_topk_ids] + local_topk_ids = _map_topk_ids_to_local_experts(local_topk_ids, expert_map) local_topk_ids = local_topk_ids.contiguous() topk_weights = topk_scores.to(torch.float32).contiguous() @@ -430,13 +433,13 @@ class MuseSpark1xFp4Experts(nn.Module): device=hidden_states.device, dtype=hidden_states.dtype, ) - cache13 = torch.empty( - (num_tokens * topk * max(gemm1_n, size_k),), + cache13 = torch.zeros( + (num_tokens * topk * (gemm1_n + size_k),), device=hidden_states.device, dtype=hidden_states.dtype, ) cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n) - cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k) + cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k) cache1 = moe_wna16_marlin_gemm( hidden_states, @@ -525,6 +528,7 @@ class MuseSpark1xFp4Experts(nn.Module): expert_outputs, local_topk_ids = self._marlin_expert_outputs( hidden_states, topk_scores, topk_ids ) + topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0) if self._post_expert_norm is not post_expert_norm: self.set_post_expert_norm(post_expert_norm) if self._post_expert_norm_weight is not None: diff --git a/python/sglang/srt/layers/quantization/gguf.py b/python/sglang/srt/layers/quantization/gguf.py index 1d8b4ba3b8..efbae869bb 100644 --- a/python/sglang/srt/layers/quantization/gguf.py +++ b/python/sglang/srt/layers/quantization/gguf.py @@ -18,6 +18,9 @@ from sglang.srt.hardware_backend.npu.quantization.moe_methods import ( from sglang.srt.hardware_backend.npu.utils import npu_format_cast from sglang.srt.layers.linear import LinearBase from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import ( + moe_align_block_size, +) from sglang.srt.layers.moe.utils import MoeRunnerBackend, get_moe_runner_backend from sglang.srt.layers.quantization.base_config import ( FusedMoEMethodBase, @@ -41,24 +44,22 @@ _is_musa = is_musa() _is_npu = is_npu() if _is_cuda: - from sgl_kernel import moe_align_block_size, moe_sum + from sgl_kernel import moe_sum from sgl_kernel.quantization import ( ggml_dequantize, ggml_moe_a8, ggml_moe_a8_vec, - ggml_moe_get_block_size, ggml_mul_mat_a8, ggml_mul_mat_vec_a8, ) from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul elif _is_musa: - from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul + from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul from sgl_kernel.quantization import ( ggml_dequantize, ggml_moe_a8, ggml_moe_a8_vec, - ggml_moe_get_block_size, ggml_mul_mat_a8, ggml_mul_mat_vec_a8, ) @@ -180,6 +181,16 @@ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES +def ggml_moe_get_block_size(qweight_type: int) -> int: + # MOE_X in the CUDA/MUSA GGUF MMQ kernels is 4. The released wheel's + # Tensor-free metadata operator has no catch-all dispatch registration. + return ( + (8 if _is_hip else 4) + if qweight_type in MMQ_QUANT_TYPES and qweight_type != WeightType.Q8_1 + else 0 + ) + + def dequantize_gguf_weight( qweight: torch.Tensor, qweight_type: int, dtype: torch.dtype ) -> torch.Tensor: @@ -239,6 +250,8 @@ def fused_moe_gguf( return gelu_and_mul(x) raise ValueError(f"Unsupported activation: {activation}") + inactive = topk_ids < 0 + x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0) out_hidden_states = torch.empty_like(x) # unless we decent expert reuse we are better off running moe_vec kernel if ( @@ -254,6 +267,11 @@ def fused_moe_gguf( sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( topk_ids, BLOCK_SIZE, E ) + # Released MMQ kernels inspect one slack block before their bound check. + block_starts = ( + torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE + ) + expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1) out = ggml_moe_a8( x, w1, @@ -265,6 +283,7 @@ def fused_moe_gguf( top_k, num_tokens, ) + out.masked_fill_(inactive.reshape(-1, 1), 0.0) out = act(out) out = ggml_moe_a8( out, @@ -277,8 +296,10 @@ def fused_moe_gguf( 1, num_tokens * top_k, ) + out.masked_fill_(inactive.reshape(-1, 1), 0.0) + valid_weights = topk_weights.masked_fill(inactive, 0.0) out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_( - topk_weights.view(num_tokens, top_k, 1) + valid_weights.view(num_tokens, top_k, 1) ) # TODO(FlamingoPg): maybe we can use moe_sum_reduce here? moe_sum(out, out_hidden_states) @@ -287,14 +308,20 @@ def fused_moe_gguf( E, N, _ = w1.shape top_k = topk_ids.shape[1] - out = ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens) + # Older vector kernels form weight addresses before checking -1. + # They have no per-expert capacity buffer; discard their dummy slots. + safe_ids = topk_ids.clamp_min(0) + out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens) + out.masked_fill_(inactive.reshape(-1, 1), 0.0) out = act(out) out = ggml_moe_a8_vec( - out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k + out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k ) + out.masked_fill_(inactive.reshape(-1, 1), 0.0) + valid_weights = topk_weights.masked_fill(inactive, 0.0) out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_( - topk_weights.view(num_tokens, top_k, 1) + valid_weights.view(num_tokens, top_k, 1) ) moe_sum(out, out_hidden_states) else: @@ -305,8 +332,10 @@ def fused_moe_gguf( ) for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)): inp = x[tok].reshape((1,) + x.shape[1:]) - current_hidden_state = None + current_hidden_state = torch.zeros_like(inp) for ww, ii in zip(w, idx): + if ii < 0: + continue expert_up = w1[ii] out = fused_mul_mat_gguf(inp, expert_up, qweight_type) @@ -316,10 +345,7 @@ def fused_moe_gguf( current_state = fused_mul_mat_gguf( out, expert_down, qweight_type2 ).mul_(ww) - if current_hidden_state is None: - current_hidden_state = current_state - else: - current_hidden_state.add_(current_state) + current_hidden_state.add_(current_state) out_hidden_states[tok] = current_hidden_state return out_hidden_states @@ -565,8 +591,6 @@ class GGUFMoEMethod(FusedMoEMethodBase): layer: torch.nn.Module, dispatch_output: StandardDispatchOutput, ) -> CombineInput: - assert self.fused_experts is None - from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput assert self.moe_runner_config.activation == "silu", ( diff --git a/python/sglang/srt/lora/lora_moe_runner_marlin.py b/python/sglang/srt/lora/lora_moe_runner_marlin.py index 405643a0ec..a7a42821b9 100644 --- a/python/sglang/srt/lora/lora_moe_runner_marlin.py +++ b/python/sglang/srt/lora/lora_moe_runner_marlin.py @@ -71,8 +71,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output - topk_weights = topk_output.topk_weights topk_ids = topk_output.topk_ids + topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0) assert runner_config.activation == "silu", "Only SiLU activation is supported." assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, ( @@ -119,9 +119,13 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): ) # Stage 1: Gate/Up (Marlin) - intermediate_cache1 = torch.empty( - (M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype + projection_cache = torch.zeros( + (M * topk * (2 * N + K),), + device=hidden_states.device, + dtype=hidden_states.dtype, ) + intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N) + intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K) intermediate_cache1 = moe_wna16_marlin_gemm( hidden_states, intermediate_cache1, @@ -165,11 +169,6 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore): silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2) # Stage 3: Down (Marlin) - intermediate_cache3 = torch.empty( - (M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype - ) - if quant_info.expert_map is not None: - intermediate_cache3.zero_() intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, diff --git a/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py b/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py index d83eff6318..736e72409a 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py +++ b/python/sglang/srt/lora/marlin_lora_temp/direct_decode.py @@ -1,7 +1,7 @@ """No-sort B200 decode kernels for Inkling routed MoE LoRA. The contract is BF16 CUDA, M<=32, top-k 6, rank 32, up to four slots, TP-local -intermediate 384 or 768, and non-EP expert IDs in ``[0, E)``. +intermediate 384 or 768, and non-EP expert IDs in ``[0, E)`` or the inactive sentinel ``-1``. """ from __future__ import annotations @@ -47,7 +47,8 @@ def _direct_gate_expand_kernel( expert = tl.load(topk_ids_ptr + route).to(tl.int64) token = route // ROUTER_TOPK slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64) - active = (slot >= 0) & (slot < NUM_SLOTS) + active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0) + expert = tl.maximum(expert, 0) offs_m = tl.arange(0, BLOCK_M) offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) offs_r = tl.arange(0, RANK) @@ -110,7 +111,8 @@ def _direct_down_shrink_kernel( expert = tl.load(topk_ids_ptr + route).to(tl.int64) token = route // ROUTER_TOPK slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64) - active = (slot >= 0) & (slot < NUM_SLOTS) + active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0) + expert = tl.maximum(expert, 0) offs_m = tl.arange(0, BLOCK_M) offs_r = tl.arange(0, RANK) offs_k = tl.arange(0, BLOCK_K) @@ -121,7 +123,7 @@ def _direct_down_shrink_kernel( k_mask = k < INTERMEDIATE_SIZE x = tl.load( activation_ptr + route * stride_xq + k[None, :] * stride_xk, - mask=(offs_m[:, None] == 0) & k_mask[None, :], + mask=active & (offs_m[:, None] == 0) & k_mask[None, :], other=0.0, ) a = tl.load( diff --git a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py index 16201396ab..0d89f3a276 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py +++ b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py @@ -232,8 +232,8 @@ def fused_experts_experimental_sgl_marlin_lora( hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output - topk_weights = topk_output.topk_weights topk_ids = topk_output.topk_ids + topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0) if ( topk_ids.ndim != 2 or topk_weights.shape != topk_ids.shape @@ -345,7 +345,7 @@ def fused_experts_experimental_sgl_marlin_lora( != lora_info.gate_up_lora_b_weights.shape[-1] ) # Allocate and prewarm on main; side streams launch kernels only. - gate_up_delta = hidden_states.new_empty((M, topk, 2 * N)) + gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N)) if not direct_decode: merged_experts_fused_moe_lora_add( output=gate_up_delta, @@ -572,9 +572,13 @@ def fused_experts_experimental_sgl_marlin_lora( ) # Stage 1: gate_up (marlin) — concurrent with the side-stream delta above. - intermediate_cache1 = torch.empty( - (M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype + projection_cache = torch.zeros( + (M * topk * (2 * N + K),), + device=hidden_states.device, + dtype=hidden_states.dtype, ) + intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N) + intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K) intermediate_cache1 = moe_wna16_marlin_gemm( hidden_states, intermediate_cache1, @@ -657,8 +661,7 @@ def fused_experts_experimental_sgl_marlin_lora( local_expert_offset=0, local_num_experts=E, zero_intermediate=( - ep_active - and lora_info.experts_shared_outer_loras + lora_info.experts_shared_outer_loras and not factored_shared_outer ), ) @@ -679,11 +682,6 @@ def fused_experts_experimental_sgl_marlin_lora( silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2) # Stage 3: down (marlin). - intermediate_cache3 = torch.empty( - (M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype - ) - if ep_active: - intermediate_cache3.zero_() intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, @@ -764,7 +762,7 @@ def fused_experts_experimental_sgl_marlin_lora( routing_cache=routing_cache, local_expert_offset=0, local_num_experts=E, - zero_intermediate=(ep_active and lora_info.experts_shared_outer_loras), + zero_intermediate=lora_info.experts_shared_outer_loras, ) # Never alias hidden_states: the shared sink still reads it. The fused B200 @@ -866,9 +864,7 @@ def fused_experts_experimental_sgl_marlin_lora( down_intermediate if down_shrink_done is not None else None ), zero_intermediate=( - ep_active - and lora_info.experts_shared_outer_loras - and down_shrink_done is None + lora_info.experts_shared_outer_loras and down_shrink_done is None ), ) diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index ef61c22ee6..20328bab29 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -835,7 +835,9 @@ class CPUGraphRunner: spec_algorithm=self.model_runner.spec_algorithm, spec_info=spec_info, capture_hidden_mode=self.capture_hidden_mode, - num_token_non_padded=self.num_token_non_padded, + num_token_non_padded=( + self.num_token_non_padded if enable_num_token_non_padded() else None + ), global_forward_mode=self.capture_forward_mode, ) self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) @@ -849,7 +851,8 @@ class CPUGraphRunner: ) with skip_ctx: with forward_context( - ForwardContext(attn_backend=self.model_runner.attn_backend) + ForwardContext(attn_backend=self.model_runner.attn_backend), + forward_batch=forward_batch, ): self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph( bs, @@ -991,11 +994,15 @@ class CPUGraphRunner: with replay_context(): with skip_ctx: prepared_forward_batch = self.prepare_replay(forward_batch, skip=skip) - output = graphs[prepared_forward_batch.batch_size]( - prepared_forward_batch.input_ids, - prepared_forward_batch.positions, - prepared_forward_batch, - ) + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend), + forward_batch=prepared_forward_batch, + ): + output = graphs[prepared_forward_batch.batch_size]( + prepared_forward_batch.input_ids, + prepared_forward_batch.positions, + prepared_forward_batch, + ) if forward_batch.batch_size in graphs: return output diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index d45a1b038e..0af731d5a2 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1143,14 +1143,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): batch.extend_input_logprob_token_ids.to(device, non_blocking=True) ) - num_tokens = len(batch.input_ids) if batch.input_ids is not None else 0 - if enable_num_token_non_padded(): - ret.global_num_token_non_padded = torch.tensor( - num_tokens, - dtype=torch.int32, - pin_memory=is_pin_memory_available(device), - ).to(device, non_blocking=True) - ret.global_num_token_non_padded_cpu = num_tokens + ret._init_num_token_non_padded( + device=device, is_draft_worker=model_runner.is_draft_worker + ) ret.init_mlp_sync_metadata( batch, device, is_draft_worker=model_runner.is_draft_worker @@ -1341,6 +1336,26 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): pin_memory=is_pin_memory_available(batch.device), ).to(batch.device, non_blocking=True) + def _init_num_token_non_padded(self, *, device, is_draft_worker: bool) -> None: + if self.input_ids is not None: + num_tokens = len(self.input_ids) + elif ( + is_draft_worker + and self.spec_info is not None + and self.forward_mode.is_decode() + ): + assert self.spec_info.num_tokens_per_req > 0 + num_tokens = self.batch_size * self.spec_info.num_tokens_per_req + else: + num_tokens = 0 + if enable_num_token_non_padded(): + self.global_num_token_non_padded = torch.tensor( + num_tokens, + dtype=torch.int32, + pin_memory=is_pin_memory_available(device), + ).to(device, non_blocking=True) + self.global_num_token_non_padded_cpu = num_tokens + def set_local_num_token_non_padded(self, *, sharded: bool) -> None: """Derive the LOCAL num_token_non_padded from the invariant GLOBAL scalar. diff --git a/python/sglang/srt/model_executor/forward_context.py b/python/sglang/srt/model_executor/forward_context.py index eb84508dfe..043bcaaa57 100644 --- a/python/sglang/srt/model_executor/forward_context.py +++ b/python/sglang/srt/model_executor/forward_context.py @@ -5,6 +5,8 @@ layer reads at depth via get_forward_context(). The only mandatory field today is attn_backend; pool refs are derived from attn_backend.* (every backend caches them at __init__), so a published ForwardContext is enough to resolve the active pools without a separate global. +The active batch lends its MoE bound getter so late preparation and TBO child +selection stay visible without retaining a parent tensor snapshot. ModelRunner._forward_raw publishes a fresh ForwardContext for the duration of each forward; callers that need a per-call override (PDmux @@ -23,12 +25,15 @@ worker threads ever share a process, migrate to contextvars.ContextVar. from __future__ import annotations from contextlib import contextmanager -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Callable, Optional if TYPE_CHECKING: + import torch + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool + from sglang.srt.model_executor.forward_batch_info import ForwardBatch @dataclass(frozen=True, slots=True) @@ -38,6 +43,8 @@ class ForwardContext: write time — use dataclasses.replace for per-call overrides.""" attn_backend: AttentionBackend + # Resolve after batch preparation; captured calls retain the static buffer. + moe_num_token_non_padded: Optional[Callable[[], Optional[torch.Tensor]]] = None _current: Optional[ForwardContext] = None @@ -76,7 +83,15 @@ def get_req_to_token_pool() -> ReqToTokenPool: @contextmanager -def forward_context(ctx: ForwardContext): +def forward_context( + ctx: ForwardContext, *, forward_batch: Optional[ForwardBatch] = None +): + if forward_batch is not None: + ctx = replace( + ctx, moe_num_token_non_padded=forward_batch.moe_num_token_non_padded + ) + elif ctx.moe_num_token_non_padded is None and _current is not None: + ctx = replace(ctx, moe_num_token_non_padded=_current.moe_num_token_non_padded) prev = set_forward_context(ctx) try: yield diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index b3bcaaf6d1..e646d19803 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -113,6 +113,7 @@ from sglang.srt.model_executor.forward_batch_info import ( from sglang.srt.model_executor.forward_context import ( ForwardContext, forward_context, + get_forward_context, has_forward_context, ) from sglang.srt.model_executor.graph_memory_usage import ( @@ -1965,9 +1966,14 @@ class ModelRunner: if prepare is not None: prepare(forward_batch) if has_forward_context(): - ctx_mgr = contextlib.nullcontext() + ctx_mgr = forward_context( + get_forward_context(), forward_batch=forward_batch + ) else: - ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend)) + ctx_mgr = forward_context( + ForwardContext(attn_backend=self.attn_backend), + forward_batch=forward_batch, + ) with ( get_model().override(shared_experts=self.model.shared_experts_config), ctx_mgr, diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 4b2c802eb2..62c130b39e 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -762,7 +762,9 @@ class BaseRunner(ABC): torch.get_device_module(mr.device).synchronize() get_parallel().tp_group.barrier() - with forward_context(ForwardContext(attn_backend=mr.attn_backend)): + with forward_context( + ForwardContext(attn_backend=mr.attn_backend), forward_batch=forward_batch + ): with run_ctx or empty_context(): run_once() diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 7793d76811..889f69360e 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -1253,7 +1253,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): # All setup hooks below read get_attn_backend() (TboForwardBatchPreparer, # DeepEP adapter, …) so they must run inside the same ForwardContext # that wraps the warmup/capture forward. - with forward_context(ForwardContext(attn_backend=attn_backend)): + with forward_context( + ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch + ): self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens) if forward_batch.lora_ids is not None: @@ -1262,60 +1264,67 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True) def run_once(): - # Backends may gather live pool mappings into their static - # metadata here; these nodes run before any model KV reads. - attn_backend.init_forward_metadata_in_graph(forward_batch) - self._record_in_graph_metadata_prep_done() - - forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = ( - None - ) - set_dp_buffer_len( - forward_batch.global_dp_buffer_len, - num_tokens, - forward_batch.dp_padding_mode.is_max_len(), - forward_batch.global_num_tokens_cpu, - ) - set_is_extend_in_batch(False) - - kwargs = {} - if ( - self.pp_size > 1 - and "pp_proxy_tensors" in inspect.signature(forward).parameters + # Debug replay retains this callback beyond the capture scope. + with forward_context( + ForwardContext(attn_backend=attn_backend), + forward_batch=forward_batch, ): - kwargs["pp_proxy_tensors"] = PPProxyTensors( - {k: v.clone() for k, v in pp_proxy_tensors.tensors.items()} - ) - if ( - ( - self.dllm_uses_input_embeds - or ( - self.model_runner.spec_algorithm.is_dflash_family() - and self.model_runner.is_draft_worker - ) - ) - and "input_embeds" in inspect.signature(forward).parameters - and not hasattr(self.model_runner.model, "forward_embed") - ): - if self.dllm_input_preparation is None: - kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens] - else: - state = self.buffer_registry.get_slot( - "dllm_input_preparation_state" - ).buffer[:num_tokens] - kwargs["input_embeds"] = self.dllm_input_preparation( - self.model_runner.model, forward_batch.input_ids, state - ) + # Backends may gather live pool mappings into their static + # metadata here; these nodes run before any model KV reads. + attn_backend.init_forward_metadata_in_graph(forward_batch) + self._record_in_graph_metadata_prep_done() - out = forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, - **kwargs, - ) - for capture_hook in self.model_runner.capture_tail_hooks: - capture_hook(self, out, forward_batch, num_tokens) - return out + forward_batch.dp_local_start_pos = ( + forward_batch.dp_local_num_tokens + ) = None + set_dp_buffer_len( + forward_batch.global_dp_buffer_len, + num_tokens, + forward_batch.dp_padding_mode.is_max_len(), + forward_batch.global_num_tokens_cpu, + ) + set_is_extend_in_batch(False) + + kwargs = {} + if ( + self.pp_size > 1 + and "pp_proxy_tensors" in inspect.signature(forward).parameters + ): + kwargs["pp_proxy_tensors"] = PPProxyTensors( + {k: v.clone() for k, v in pp_proxy_tensors.tensors.items()} + ) + if ( + ( + self.dllm_uses_input_embeds + or ( + self.model_runner.spec_algorithm.is_dflash_family() + and self.model_runner.is_draft_worker + ) + ) + and "input_embeds" in inspect.signature(forward).parameters + and not hasattr(self.model_runner.model, "forward_embed") + ): + if self.dllm_input_preparation is None: + kwargs["input_embeds"] = self.buffers.input_embeds[ + :num_tokens + ] + else: + state = self.buffer_registry.get_slot( + "dllm_input_preparation_state" + ).buffer[:num_tokens] + kwargs["input_embeds"] = self.dllm_input_preparation( + self.model_runner.model, forward_batch.input_ids, state + ) + + out = forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + **kwargs, + ) + for capture_hook in self.model_runner.capture_tail_hooks: + capture_hook(self, out, forward_batch, num_tokens) + return out self.deepep_adapter.capture(is_extend_in_batch=False) canary_ctx = ( diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 7e0f3b3e14..a598fc7637 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -46,6 +46,7 @@ from sglang.srt.model_executor.forward_batch_info import ( from sglang.srt.model_executor.forward_context import ( ForwardContext, forward_context, + get_forward_context, get_req_to_token_pool, get_token_to_kv_pool, ) @@ -259,7 +260,11 @@ class EagerRunner(BaseRunner): ctx = device_timer_ctx(model_runner.device_timer, "decode") - with ctx, pdmux_ctx: + with ( + ctx, + pdmux_ctx, + forward_context(get_forward_context(), forward_batch=forward_batch), + ): return model_runner.model.forward( forward_batch.input_ids, forward_batch.positions, @@ -341,7 +346,10 @@ class EagerRunner(BaseRunner): if forward_batch.forward_mode.is_target_verify() else "extend" ) - with device_timer_ctx(model_runner.device_timer, category): + with ( + device_timer_ctx(model_runner.device_timer, category), + forward_context(get_forward_context(), forward_batch=forward_batch), + ): if cp_active and ( not get_parallel().enable_cp_tp_group_sharing or getattr(model_runner.model, "prepare_cp_inputs", None) is not None @@ -448,7 +456,10 @@ class EagerRunner(BaseRunner): model_runner.attn_backend.forward_metadata = None kwargs = model_runner._pp_kwargs(pp_proxy_tensors) - with device_timer_ctx(model_runner.device_timer, "idle"): + with ( + device_timer_ctx(model_runner.device_timer, "idle"), + forward_context(get_forward_context(), forward_batch=forward_batch), + ): return model_runner.model.forward( forward_batch.input_ids, forward_batch.positions, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 320bb662fd..fc7e93b9f2 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -819,7 +819,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): full_graph=self._is_full_backend, raw_num_tokens=raw_num_tokens ), forward_context( - ForwardContext(attn_backend=self.model_runner.attn_backend) + ForwardContext(attn_backend=self.model_runner.attn_backend), + forward_batch=forward_batch, ), set_tc_piecewise_forward_context( forward_batch, @@ -953,7 +954,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): set_forward_batch(fb), prefill_graph_scope(full_graph=self._is_full_backend), forward_context( - ForwardContext(attn_backend=self.model_runner.attn_backend) + ForwardContext(attn_backend=self.model_runner.attn_backend), + forward_batch=fb, ), set_tc_piecewise_forward_context( fb, @@ -1234,7 +1236,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): and stash the returned per-bucket metadata object; otherwise fall back to the generic eager init that BCG/TC_PIECEWISE use today.""" attn_backend = self.model_runner.attn_backend - with forward_context(ForwardContext(attn_backend=attn_backend)): + with forward_context( + ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch + ): if not self.use_captured_attn_metadata: attn_backend.init_forward_metadata(forward_batch) return diff --git a/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py index b2b4d9bc3a..bcf92ae34b 100644 --- a/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/row_bucket_graph_runner.py @@ -20,7 +20,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( set_graph_pool_id, ) from sglang.srt.distributed.parallel_state import graph_capture -from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.model_executor.forward_context import ( + forward_context, + get_attn_backend, + get_forward_context, + has_forward_context, +) from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( BreakableCUDAGraph, BreakableCUDAGraphCapture, @@ -120,7 +125,14 @@ class RowBucketGraphRunner: # The eager breaks read the step's real row count off this view. step_batch = copy.copy(forward_batch) step_batch.global_num_token_non_padded_cpu = num_rows - with context(step_batch): + with ( + ( + forward_context(get_forward_context(), forward_batch=step_batch) + if has_forward_context() + else nullcontext() + ), + context(step_batch), + ): if (rows, graph_key) not in self._graphs: # Largest first, so smaller buckets reuse the pool memory the # larger ones took; a bucket below this step's rows cannot be @@ -168,7 +180,14 @@ class RowBucketGraphRunner: forward_batch.num_token_non_padded, num_rows ) capture_batch.num_token_non_padded = num_token_non_padded - with batch_scope(capture_batch, inputs): + with ( + ( + forward_context(get_forward_context(), forward_batch=capture_batch) + if has_forward_context() + else nullcontext() + ), + batch_scope(capture_batch, inputs), + ): graph = self._record( lambda: body(inputs, capture_batch), rows, diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 6c3857a0d7..befac009b7 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -29,7 +29,6 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, @@ -492,10 +491,6 @@ class BailingMoESparseMoeBlock(nn.Module): hidden_states, router_logits, dynamic_expert_bias=dynamic_expert_bias, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 6002c8ac74..7dfcaa34ab 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -21,7 +21,6 @@ from sglang.srt.configs import KimiLinearConfig from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList @@ -832,10 +831,6 @@ class BailingMoE(nn.Module): hidden_states, router_logits, dynamic_expert_bias=dynamic_expert_bias, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) final_hidden_states = self.experts(hidden_states, topk_output) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index bf2d2ace7e..60c7f4a27f 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -66,7 +66,6 @@ from sglang.srt.distributed import divide from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.amx_utils import PackWeightMethod @@ -120,10 +119,10 @@ from sglang.srt.layers.moe.token_dispatcher.base import ( DispatchOutput, ) from sglang.srt.layers.moe.topk import ( - BypassedTopKOutput, TopK, TopKOutputFormat, fused_shared_expert_weight, + is_expert_remapping_enabled, is_routed_scaling_factor_fused_in_topk, ) from sglang.srt.layers.moe.utils import ( @@ -1038,7 +1037,8 @@ class DeepseekV2MoE(nn.Module): and not self._enable_a2a_moe and not self._fuse_shared_experts_inside_sbo and not getattr(self, "is_hash", False) - and not get_exec().moe.enable_eplb + and not is_expert_remapping_enabled() + and get_exec().moe.ep_dispatch_algorithm not in ("fake", "lp") ) def forward( @@ -1067,11 +1067,6 @@ class DeepseekV2MoE(nn.Module): input_ids_global=input_ids_global, ) - num_token_non_padded = ( - forward_batch.moe_num_token_non_padded() - if forward_batch is not None - else None - ) use_vision_topk = self.gate.e_score_correction_bias_vl is not None if use_vision_topk and _is_hip: use_vision_topk = _hip_moe.batch_has_images(forward_batch) @@ -1109,7 +1104,6 @@ class DeepseekV2MoE(nn.Module): gemm_output_zero_allocator, input_ids, input_ids_global=input_ids_global, - num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, return_moe_output=return_moe_output, leave_output_sum=leave_output_sum, @@ -1121,7 +1115,6 @@ class DeepseekV2MoE(nn.Module): input_ids, input_ids_global=input_ids_global, skip_shared_experts=skip_shared_experts, - num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, return_moe_output=return_moe_output, leave_output_sum=leave_output_sum, @@ -1137,7 +1130,6 @@ class DeepseekV2MoE(nn.Module): gemm_output_zero_allocator: BumpAllocator = None, input_ids: Optional[torch.Tensor] = None, input_ids_global: Optional[torch.Tensor] = None, - num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, return_moe_output: bool = False, leave_output_sum: bool = False, @@ -1167,11 +1159,6 @@ class DeepseekV2MoE(nn.Module): has_shared_output = ( hidden_states.shape[0] > 0 and self.num_fused_shared_experts == 0 ) - dispatch_info = ( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if get_exec().moe.enable_eplb and not self.is_nextn - else None - ) # router_logits: (num_tokens, n_experts) if _is_hip: @@ -1185,11 +1172,7 @@ class DeepseekV2MoE(nn.Module): router_logits = self.gate(hidden_states, gemm_output_zero_allocator) router_logits_partials = None if use_flashinfer_trtllm_bypass: - topk_output = BypassedTopKOutput( - hidden_states=hidden_states, - router_logits=router_logits, - topk_config=self.topk.topk_config, - ) + topk_output = self.topk.forward_cuda(hidden_states, router_logits) else: topk_kwargs = ( {"input_ids": input_ids_global} @@ -1203,14 +1186,11 @@ class DeepseekV2MoE(nn.Module): self, router_logits, input_ids_global, - num_token_non_padded=num_token_non_padded, ) else: topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=dispatch_info, **topk_kwargs, ) # Issued after the router so the main chain stays on the main stream at replay. @@ -1323,7 +1303,6 @@ class DeepseekV2MoE(nn.Module): input_ids: Optional[torch.Tensor] = None, input_ids_global: Optional[torch.Tensor] = None, skip_shared_experts: bool = False, - num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, return_moe_output: bool = False, leave_output_sum: bool = False, @@ -1332,11 +1311,6 @@ class DeepseekV2MoE(nn.Module): self.shared_experts.gate_up_proj ): return self.forward_cpu(hidden_states) - dispatch_info = ( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if get_exec().moe.enable_eplb and not self.is_nextn - else None - ) defer_shared = not self.experts.moe_runner_config.inplace # PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): shared expert is computed on the LOCAL # hidden in the decoder layer (before the dp gather) and added after the @@ -1383,21 +1357,18 @@ class DeepseekV2MoE(nn.Module): self, router_logits, input_ids_global, - num_token_non_padded=num_token_non_padded, ) else: topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=dispatch_info, **topk_kwargs, ) else: pre_quant_input = None shared_output = None topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) if self._fuse_shared_experts_inside_sbo and not skip_shared_experts: @@ -1591,19 +1562,11 @@ class DeepseekV2MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ) - if not self.is_nextn - else None - ), **topk_kwargs, ) else: topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) if sbo_overlap_dispatch_flag: @@ -1979,19 +1942,11 @@ class DeepseekV2MoE(nn.Module): state.topk_output = self.topk( hidden_states=hidden_states, router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ) - if not self.is_nextn - else None - ), **topk_kwargs, ) else: state.topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) def op_dispatch_a(self, state): diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index cf4ed28c87..6b79e3efce 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -28,7 +28,12 @@ from sglang.srt.model_executor.cuda_graph_config import ( check_cuda_graph_backend, ) from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.model_executor.forward_context import ( + forward_context, + get_attn_backend, + get_forward_context, + has_forward_context, +) from sglang.srt.model_executor.model_runner_components.layer_setup import ( compute_attention_and_moe_layers, ) @@ -163,6 +168,15 @@ class DecoderReplayTail: max_rows=max_rows, ) + @staticmethod + def _can_capture_moe_bound(forward_batch: ForwardBatch) -> bool: + # The row runner replaces only the local count with a graph-owned buffer. + bound = forward_batch.moe_num_token_non_padded() + return bound is None or ( + bound is forward_batch.num_token_non_padded + and bound is not forward_batch.global_num_token_non_padded + ) + def plan( self, tail: Optional[LateLayerTail], @@ -199,6 +213,7 @@ class DecoderReplayTail: or get_global_expert_distribution_recorder().recording or not self.runner.warm or (tail is not None and tail.cp_metadata is not None) + or not self._can_capture_moe_bound(forward_batch) ): return False, None if not dp: @@ -414,7 +429,14 @@ class DecoderReplayTail: gathered = input_ids.new_empty((sum(dp_rows), 1)) dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch) input_ids_global = gathered.squeeze(-1) - with self._break_context(tail_batch): + with ( + ( + forward_context(get_forward_context(), forward_batch=tail_batch) + if has_forward_context() + else nullcontext() + ), + self._break_context(tail_batch), + ): return self._layer_stack( state, positions=positions, @@ -681,6 +703,7 @@ def _tail_layers_in_prefill_graph( if ( not get_flags().dp.capturing_prefill_graph and runner.warm + and tail_layers._can_capture_moe_bound(forward_batch) and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None ): residual, pre, aux = tail_layers.run(**kwargs) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 4895e6b878..9ab504b0da 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -51,7 +51,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer @@ -467,14 +466,10 @@ class Dots3MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) final_hidden_states = self.experts( diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index a3bde21cb0..288ce47315 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -26,7 +26,6 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -228,10 +227,6 @@ class ExaoneMoESparseMoEBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 234985fa14..d48aac3afb 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -36,7 +36,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import ( @@ -653,10 +652,6 @@ class Glm4MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index a23bbf6b38..8a07e9de49 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -30,7 +30,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import ( @@ -377,10 +376,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index ecc75ed52a..2ea986414f 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -34,7 +34,6 @@ from triton.language.extra import libdevice from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -918,10 +917,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 63c5e490c3..88cd872abf 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -97,6 +97,7 @@ from sglang.srt.model_loader.utils import ( from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA from sglang.srt.runtime_context import ( + get_exec, get_parallel, get_stream, ) @@ -251,6 +252,15 @@ class LongcatFlashMoE(nn.Module): self.top_k = config.moe_topk self.zero_expert_num = config.zero_expert_num self.zero_expert_type = config.zero_expert_type + if ( + self.zero_expert_num > 0 + and get_exec().moe.ep_dispatch_algorithm is not None + ): + raise ValueError( + "LongCat zero experts do not support expert placement remapping. " + "Disable EPLB, redundant experts, custom expert placement, and " + "--ep-dispatch-algorithm for this model." + ) if config.rounter_params_dtype == "float32": self.rounter_params_dtype = torch.float32 diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 8a7e3a65be..f197f37c89 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -29,7 +29,6 @@ from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, @@ -559,11 +558,6 @@ class MiMoV2MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if self._uses_expert_remap - else None - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) @@ -595,10 +589,6 @@ class MiMoV2MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) @@ -628,10 +618,6 @@ class MiMoV2MoE(nn.Module): state.topk_output = self.topk( hidden_states=hidden_states, router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: state.topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 6e94ec1319..b9c94f3b90 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -29,7 +29,6 @@ from transformers import PretrainedConfig from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, @@ -614,10 +613,6 @@ class MiniMaxM2MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(device=hidden_states.device) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 1e82cf9ceb..f40c67c0e2 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -36,7 +36,6 @@ from sglang.srt.configs.model_config import ( ) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, @@ -535,10 +534,6 @@ class MiniMaxM3MoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 8847bd2955..73cfcb8a79 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -29,7 +29,6 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import ReLU2 from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( HybridLinearAttnBackend, @@ -349,7 +348,6 @@ class NemotronHMoE(nn.Module): def _forward_core( self, hidden_states: torch.Tensor, - num_token_non_padded: torch.Tensor | None, forward_batch: ForwardBatch | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: if ( @@ -364,21 +362,16 @@ class NemotronHMoE(nn.Module): and is_in_breakable_cuda_graph() and is_sm120() ): - return self._forward_core_prefill_overlap( - hidden_states, num_token_non_padded - ) + return self._forward_core_prefill_overlap(hidden_states) if _is_cuda and ( not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode() ): - return self._forward_core_shared_routed_overlap( - hidden_states, num_token_non_padded - ) - return self._forward_core_normal(hidden_states, num_token_non_padded) + return self._forward_core_shared_routed_overlap(hidden_states) + return self._forward_core_normal(hidden_states) def _forward_core_normal( self, hidden_states: torch.Tensor, - num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: # router_scores: [num_tokens, num_experts] # bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk. @@ -392,12 +385,6 @@ class NemotronHMoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if not self.is_nextn - else None - ), ) if self.use_latent_moe: hidden_states = self._apply_fc1_latent_proj(hidden_states) @@ -407,7 +394,6 @@ class NemotronHMoE(nn.Module): def _forward_core_shared_routed_overlap( self, hidden_states: torch.Tensor, - num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: alt_stream = _get_or_create_alt_stream(self.device_module) @@ -427,12 +413,6 @@ class NemotronHMoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if not self.is_nextn - else None - ), ) if self.use_latent_moe: hidden_states = self._apply_fc1_latent_proj(hidden_states) @@ -444,7 +424,6 @@ class NemotronHMoE(nn.Module): def _forward_core_prefill_overlap( self, hidden_states: torch.Tensor, - num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor]: # A multi-stream graph per hybrid segment creates one internal stream # per executable. Keep each graph linear and launch shared experts on @@ -460,12 +439,6 @@ class NemotronHMoE(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if not self.is_nextn - else None - ), ) if self.use_latent_moe: hidden_states = self._apply_fc1_latent_proj(hidden_states) @@ -509,11 +482,6 @@ class NemotronHMoE(nn.Module): # MoE runner / topk), so final_hidden_states is already scaled. final_hidden_states, shared_output = self._forward_core( hidden_states, - num_token_non_padded=( - forward_batch.moe_num_token_non_padded() - if forward_batch is not None - else None - ), forward_batch=forward_batch, ) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index d30064f28f..c22f96b2c4 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -38,7 +38,6 @@ from sglang.srt.distributed import ( from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, @@ -611,14 +610,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ) - if not self.is_nextn - else None - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) @@ -667,16 +658,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=( - forward_batch.moe_num_token_non_padded() - if forward_batch is not None - else None - ), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if not self.is_nextn - else None - ), ) if self.enable_shared_expert_fusion: topk_output = self._append_shared_to_topk_output( diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index eab3a112bc..41efdad5cf 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -28,7 +28,6 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator from sglang.srt.layers.layer_boundary import ( append_stages, @@ -342,20 +341,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module): num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) - dispatch_info = ( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - if get_exec().moe.enable_eplb - else None - ) - if hidden_states.shape[0] > 0: # router_logits: (num_tokens, n_experts) router_logits, _ = self.gate(hidden_states) - topk_output = self.topk( - hidden_states, - router_logits, - expert_location_dispatch_info=dispatch_info, - ) + topk_output = self.topk(hidden_states, router_logits) else: topk_output = self.topk.empty_topk_output(hidden_states.device) final_hidden_states = self.experts(hidden_states, topk_output) @@ -370,10 +359,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) @@ -408,14 +393,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=( - forward_batch.moe_num_token_non_padded() - if forward_batch is not None - else None - ), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) assert TopKOutputChecker.format_is_standard(topk_output), ( "MegaMoE pre-dispatch consumes raw topk ids/weights; " @@ -454,10 +431,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module): state.topk_output = self.topk( hidden_states=hidden_states, router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: state.topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 77126a789f..f1b9ce6a52 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -13,7 +13,6 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -159,10 +158,6 @@ class SDARMoeSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index d9918b826f..fa55d85cce 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -6,7 +6,6 @@ import torch.nn.functional as F from torch import nn from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -256,10 +255,6 @@ class Step3p5MoEMLP(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 7f0d0ad61e..ba489639df 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -38,7 +38,6 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import ( @@ -1000,10 +999,6 @@ class XllmSparseMoeBlock(nn.Module): topk_output = self.topk( hidden_states, router_logits, - num_token_non_padded=forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=( - ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) - ), ) # Apply router scaling factor after renormalization if self.router_scaling_factor != 1.0: @@ -1014,7 +1009,7 @@ class XllmSparseMoeBlock(nn.Module): topk_output.topk_weights = scaled_weights else: topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) final_hidden_states = self.experts( hidden_states=hidden_states, @@ -1051,7 +1046,7 @@ class XllmSparseMoeBlock(nn.Module): if hidden_states.shape[0] == 0: shared_output = None topk_output = self.topk.empty_topk_output( - hidden_states.device, layer_id=self.layer_id + hidden_states.device, ) final_hidden_states = self.experts(hidden_states, topk_output) else: diff --git a/python/sglang/srt/multimodal/dsv41/vl_routing.py b/python/sglang/srt/multimodal/dsv41/vl_routing.py index 747b030093..5c0a281d88 100644 --- a/python/sglang/srt/multimodal/dsv41/vl_routing.py +++ b/python/sglang/srt/multimodal/dsv41/vl_routing.py @@ -2,6 +2,7 @@ import torch import torch.nn.functional as F from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location_dispatch import transform_select_experts_inputs from sglang.srt.layers.moe.topk import ( _RENORMALIZE_SUM_EPSILON, StandardTopKOutput, @@ -12,19 +13,10 @@ from sglang.srt.layers.moe.topk import ( ) from sglang.srt.layers.moe.utils import ( get_moe_a2a_backend, - has_per_rank_fused_shared_slots, ) from sglang.srt.utils import is_cuda -def _scale_fused_shared_weights(weights, num_fused_shared_experts, scaling_factor): - # Standard EP replicates the fused shared expert on every rank and all-reduces, - # so the shared columns carry a 1/ep_size factor. - if num_fused_shared_experts and scaling_factor is not None: - weights[:, -num_fused_shared_experts:] *= scaling_factor - return weights - - def vision_topk( moe, logits, @@ -32,25 +24,35 @@ def vision_topk( num_token_non_padded=None, expert_location_dispatch_info=None, ): + bound, dispatch_info = moe.topk.routing_inputs() + if num_token_non_padded is None: + num_token_non_padded = bound + if expert_location_dispatch_info is None: + expert_location_dispatch_info = dispatch_info config = moe.topk.topk_config + logits, correction_bias = transform_select_experts_inputs( + logits, moe.gate.e_score_correction_bias, expert_location_dispatch_info + ) + correction_bias_vl = moe.gate.e_score_correction_bias_vl + if ( + expert_location_dispatch_info is not None + and expert_location_dispatch_info.ep_dispatch_algorithm == "fake" + ): + correction_bias_vl = torch.zeros_like(correction_bias_vl) num_shared_topk_slots = config.num_shared_topk_slots use_mega_moe = get_moe_a2a_backend().is_megamoe() - if num_shared_topk_slots and not use_mega_moe: - # This path bypasses _post_process_topk_ids, which appends the per-rank slots. - assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), ( - "VL routing does not support per-rank fused shared slots" - ) if is_cuda(): from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate - from sglang.srt.layers.moe.utils import get_moe_runner_backend # Same admission as _fused_gate_emits_packed_ids: the shared-expert slots # rescaled below would rewrite weights after the router. packed_topk = None if ( num_shared_topk_slots == 0 + and expert_location_dispatch_info is None and not use_mega_moe - and get_moe_runner_backend().is_flashinfer_mxfp4() + and config.runner_backend is not None + and config.runner_backend.is_flashinfer_mxfp4() ): packed_topk = torch.empty( (logits.shape[0], config.top_k), dtype=torch.int32, device=logits.device @@ -58,11 +60,11 @@ def vision_topk( weights, indices = moe_fused_gate( logits, - moe.gate.e_score_correction_bias, + correction_bias, topk=config.top_k, scoring_func="sqrtsoftplus", num_fused_shared_experts=num_shared_topk_slots, - bias_alt=moe.gate.e_score_correction_bias_vl, + bias_alt=correction_bias_vl, input_ids=input_ids, bias_alt_token_id=moe.config.image_token_id, renormalize=config.renormalize and config.top_k > 1, @@ -87,12 +89,12 @@ def vision_topk( return StandardTopKOutput(weights, indices, logits) scores = F.softplus(logits.float()).sqrt() if input_ids is None: - bias = moe.gate.e_score_correction_bias + bias = correction_bias else: bias = torch.where( (input_ids == moe.config.image_token_id)[:, None], - moe.gate.e_score_correction_bias_vl, - moe.gate.e_score_correction_bias, + correction_bias_vl, + correction_bias, ) # The shared slots appended below use the same layout as biased_grouped_topk_gpu. topk_routed = config.top_k - num_shared_topk_slots @@ -136,28 +138,19 @@ def _finish_vision_topk( ): config = moe.topk.topk_config use_mega_moe = get_moe_a2a_backend().is_megamoe() - if use_mega_moe: - # Use the same EPLB mapping, recording and home-rank shared slots as - # ordinary top-k before handing physical expert IDs to MegaMoE. - indices, weights, recorder_ids = _post_process_topk_ids( - indices, - weights, - config, - logits, - moe.layer_id, - num_token_non_padded=num_token_non_padded, - expert_location_dispatch_info=expert_location_dispatch_info, - padded_rows_masked=padded_rows_masked, - ) - if recorder_ids is not None: - get_global_expert_distribution_recorder().on_select_experts( - topk_ids=recorder_ids - ) - else: - weights = _scale_fused_shared_weights( - weights, - config.num_shared_topk_slots, - config.fused_shared_experts_scaling_factor, + indices, weights, recorder_ids = _post_process_topk_ids( + indices, + weights, + config, + logits, + moe.layer_id, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + padded_rows_masked=padded_rows_masked, + ) + if recorder_ids is not None: + get_global_expert_distribution_recorder().on_select_experts( + topk_ids=recorder_ids ) # Shared-slot remapping can overwrite padded weights; mask last so idle # and partially padded DP ranks never dispatch a phantom shared expert. diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 85caa24e9f..f7205cae82 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -511,7 +511,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1) return ret - with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)): + with forward_context( + ForwardContext(attn_backend=self.draft_attn_backend), + forward_batch=forward_batch, + ): self.draft_attn_backend.init_forward_metadata_out_graph( forward_batch, in_capture=True ) diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index 663fd2773e..9a3f7b7a82 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -482,7 +482,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): return ret with forward_context( - ForwardContext(attn_backend=self.draft_extend_attn_backend) + ForwardContext(attn_backend=self.draft_extend_attn_backend), + forward_batch=forward_batch, ): self.draft_extend_attn_backend.init_forward_metadata_out_graph( forward_batch, in_capture=True diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index f1d5cea6f8..bf29c06429 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -800,7 +800,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): forward_context( ForwardContext( attn_backend=self.draft_attn_backend.attn_backends[i] - ) + ), + forward_batch=forward_batch, ), canary_index_ctx, ): @@ -903,7 +904,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): forward_context( ForwardContext( attn_backend=self.draft_attn_backend.attn_backends[i] - ) + ), + forward_batch=forward_batch, ), canary_index_ctx, ): diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index 773abb1650..a5fb3f643a 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -357,7 +357,10 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner): saved_backend_pool = self.draft_attn_backend.token_to_kv_pool self.draft_attn_backend.token_to_kv_pool = target_pool try: - with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)): + with forward_context( + ForwardContext(attn_backend=self.draft_attn_backend), + forward_batch=forward_batch, + ): self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph( forward_batch ) diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py index b2ab12c00c..3527bbc3ed 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py @@ -565,7 +565,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): with ( self._target_kv_pool_view(forward_batch), - forward_context(ForwardContext(attn_backend=self.draft_attn_backend)), + forward_context( + ForwardContext(attn_backend=self.draft_attn_backend), + forward_batch=forward_batch, + ), ): seed_output = self.draft_model_runner.forward(forward_batch).logits_output @@ -608,7 +611,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): with ( self._target_kv_pool_view(forward_batch), - forward_context(ForwardContext(attn_backend=self.draft_attn_backend)), + forward_context( + ForwardContext(attn_backend=self.draft_attn_backend), + forward_batch=forward_batch, + ), ): logits_output = self.draft_model_runner.forward( forward_batch diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 3a92275839..bf3ab0a44e 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -451,7 +451,9 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): attn_backend.init_forward_metadata_in_graph(forward_batch) return self._run_step_body(forward_batch, num_tokens, bs) - with forward_context(ForwardContext(attn_backend=attn_backend)): + with forward_context( + ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch + ): attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True) self.deepep_adapter.capture(is_extend_in_batch=True) shape_key = self._make_graph_key(bs) @@ -1026,7 +1028,9 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner( forward_batch = r.get_forward_batch(bs) forward_batch = r._postprocess_forward_batch(forward_batch, bs) attn_backend = self.draft_extend_attn_backend_list[r.step] - with forward_context(ForwardContext(attn_backend=attn_backend)): + with forward_context( + ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch + ): attn_backend.init_forward_metadata_out_graph( forward_batch, in_capture=True ) @@ -1037,7 +1041,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner( outs = [] for i, (r, forward_batch, num_tokens, attn_backend) in enumerate(items): attn_backend.init_forward_metadata_in_graph(forward_batch) - with forward_context(ForwardContext(attn_backend=attn_backend)): + with forward_context( + ForwardContext(attn_backend=attn_backend), + forward_batch=forward_batch, + ): ret = r._run_step_body(forward_batch, num_tokens, bs) outs.append(ret) if i < n - 1: diff --git a/python/sglang/srt/speculative/uno_worker_v2.py b/python/sglang/srt/speculative/uno_worker_v2.py index d5aff14fe9..b82b67740b 100644 --- a/python/sglang/srt/speculative/uno_worker_v2.py +++ b/python/sglang/srt/speculative/uno_worker_v2.py @@ -814,7 +814,10 @@ class UnoWorkerV2(BaseSpecWorker): else contextlib.nullcontext() ) attn_context = ( - forward_context(ForwardContext(attn_backend=self._uno_draft_attn_backend)) + forward_context( + ForwardContext(attn_backend=self._uno_draft_attn_backend), + forward_batch=forward_batch, + ) if self.tree_mode else contextlib.nullcontext() ) diff --git a/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py b/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py new file mode 100644 index 0000000000..d21a8a8395 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py @@ -0,0 +1,274 @@ +"""CUTLASS FP8 maps and reductions skip inactive expert routes.""" + +import unittest +from collections import defaultdict +from unittest.mock import patch + +import torch + +from sglang.kernels.ops.moe.shuffle_weighted_sum import ( + _shuffle_weighted_sum_kernel, + shuffle_weighted_sum, +) +from sglang.srt.layers.moe import cutlass_moe +from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=12, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestCutlassFp8InactiveRoutes(CustomTestCase): + def test_masked_reduction_ignores_poisoned_slots(self): + for dtype in (torch.float16, torch.bfloat16): + with self.subTest(dtype=dtype): + inputs = torch.full((6, 257), float("nan"), device="cuda", dtype=dtype) + inputs[0].fill_(2) + inputs[1].fill_(4) + mapping = torch.tensor( + [0, 1, -1, -1, 0, -1], device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [[0.25, 0.75], [float("nan"), float("nan")], [0.5, float("nan")]], + device="cuda", + dtype=dtype, + ) + output = torch.empty((3, 257), device="cuda", dtype=dtype) + shuffle_weighted_sum(inputs, output, mapping, weights) + expected = torch.tensor([3.5, 0, 1], device="cuda", dtype=dtype)[ + :, None + ].expand_as(output) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + def test_reduction_reuses_kernels_across_token_counts(self): + """New lengths reuse the reduction and preserve mixed/empty route results.""" + kernel = _shuffle_weighted_sum_kernel + lengths = (1, 3, 7, 8, 15, 16, 17, 31, 32, 33, 64, 65) + for dtype in (torch.float16, torch.bfloat16): + with ( + self.subTest(dtype=dtype), + patch.object( + kernel, "device_caches", defaultdict(kernel.create_binder) + ), + ): + cache = kernel.device_caches[torch.cuda.current_device()][0] + specialization_count = None + for strided_weights in (False, True): + for tokens in lengths: + with self.subTest( + tokens=tokens, strided_weights=strided_weights + ): + copies = (tokens + 2) // 3 + inputs = torch.full( + (tokens * 2, 257), + float("nan"), + device="cuda", + dtype=dtype, + ) + inputs[0].fill_(2) + inputs[1].fill_(4) + mapping = ( + torch.tensor( + [[0, 1], [1, -1], [-1, -1]], + device="cuda", + dtype=torch.int32, + ) + .repeat(copies, 1)[:tokens] + .flatten() + ) + weights = torch.tensor( + [ + [0.25, 0.75], + [0.5, float("nan")], + [float("nan"), float("nan")], + ], + device="cuda", + dtype=dtype, + ).repeat(copies, 1)[:tokens] + if strided_weights: + weights = weights.t().contiguous().t() + output = torch.empty( + (tokens, 257), device="cuda", dtype=dtype + ) + shuffle_weighted_sum(inputs, output, mapping, weights) + expected = ( + torch.tensor([3.5, 2, 0], device="cuda", dtype=dtype) + .repeat(copies)[:tokens, None] + .expand_as(output) + ) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + if specialization_count is None: + specialization_count = len(cache) + self.assertEqual(len(cache), specialization_count) + + mapping.fill_(-1) + inputs.fill_(float("nan")) + weights.fill_(float("nan")) + output.fill_(float("nan")) + shuffle_weighted_sum(inputs, output, mapping, weights) + self.assertTrue(bool(torch.isfinite(output).all())) + self.assertEqual(int(torch.count_nonzero(output)), 0) + self.assertEqual(len(cache), specialization_count) + + def test_native_fp8_padding_survives_graph_replay(self): + if torch.cuda.get_device_capability()[0] not in (9, 10, 12): + self.skipTest("CUTLASS block FP8 requires SM90, SM100 or SM120") + for mxfp8 in (False, True): + if mxfp8 and torch.cuda.get_device_capability()[0] != 10: + continue + with self.subTest(mxfp8=mxfp8): + self._check_native(mxfp8) + + def _check_native(self, mxfp8): + experts, tokens, hidden, intermediate = 2, 16, 256, 256 + generator = torch.Generator(device="cuda").manual_seed(945) + inputs = ( + torch.randn( + (tokens, hidden), + generator=generator, + device="cuda", + dtype=torch.bfloat16, + ) + / 8 + ) + ids = torch.tensor( + [[0, 1], [1, -1], [-1, -1], [1, 0]] * 4, device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [ + [0.25, 0.75], + [0.6, float("nan")], + [float("nan"), float("nan")], + [0.8, 0.2], + ] + * 4, + device="cuda", + ) + inputs[(ids < 0).all(dim=-1)] = float("nan") + w1 = ( + ( + torch.randn( + (experts, 2 * intermediate, hidden), + generator=generator, + device="cuda", + ) + / 16 + ) + .to(torch.float8_e4m3fn) + .transpose(1, 2) + ) + w2 = ( + ( + torch.randn( + (experts, hidden, intermediate), generator=generator, device="cuda" + ) + / 16 + ) + .to(torch.float8_e4m3fn) + .transpose(1, 2) + ) + if mxfp8: + scale1 = torch.full( + (experts, hidden // 32, 2 * intermediate), + 127, + device="cuda", + dtype=torch.uint8, + ) + scale2 = torch.full( + (experts, intermediate // 32, hidden), + 127, + device="cuda", + dtype=torch.uint8, + ) + else: + scale1 = torch.ones( + (experts, 2 * intermediate // 128, hidden // 128), device="cuda" + ).transpose(1, 2) + scale2 = torch.ones( + (experts, hidden // 128, intermediate // 128), device="cuda" + ).transpose(1, 2) + params = CutlassMoEParams( + CutlassMoEType.BlockscaledFP8, + torch.device("cuda"), + experts, + intermediate, + hidden, + ) + workspace = torch.empty(32 * 1024 * 1024, device="cuda", dtype=torch.uint8) + + def run(x, route_ids, route_weights): + return cutlass_moe.cutlass_fused_experts_fp8( + x, + w1, + w2, + scale1, + scale2, + route_weights, + route_ids, + params.ab_strides_13, + params.c_strides_13, + params.ab_strides_2, + params.c_strides_2, + workspace, + params.a_ptrs, + params.b_ptrs, + params.out_ptrs, + params.a_scales_ptrs, + params.b_scales_ptrs, + params.expert_offsets, + params.problem_sizes1, + params.problem_sizes2, + use_mxfp8=mxfp8, + enable_es=(mxfp8, mxfp8), + ) + + reference = run(inputs.nan_to_num(), ids.clamp_min(0), weights.nan_to_num()) + prepare = cutlass_moe.prepare_moe_input + + def checked_prepare(*args): + prepare(*args) + route_ids, offsets, _, _, a_map, c_map = args[:6] + self.assertTrue(bool(((a_map >= 0) & (a_map < tokens)).all())) + self.assertTrue(bool((c_map[route_ids.flatten() < 0] == -1).all())) + self.assertEqual(int(offsets[-1]), int((route_ids >= 0).sum())) + + original_empty = torch.empty + + def poisoned_empty(*args, **kwargs): + tensor = original_empty(*args, **kwargs) + if tensor.is_floating_point(): + tensor.fill_(float("nan")) + elif tensor.dtype == torch.int32: + tensor.fill_(-999) + return tensor + + # The pre-fix map allocation fails checked_prepare before an unsafe gather. + with ( + patch.object(torch, "empty", poisoned_empty), + patch.object(cutlass_moe, "prepare_moe_input", checked_prepare), + ): + actual = run(inputs, ids, weights) + torch.testing.assert_close(actual, reference, rtol=0.025, atol=0.01) + self.assertTrue(bool(torch.isfinite(actual).all())) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + result = run(inputs, ids, weights) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + result = run(inputs, ids, weights) + graph.replay() + torch.testing.assert_close(result, reference, rtol=0.025, atol=0.01) + ids.fill_(-1) + inputs.fill_(float("nan")) + weights.fill_(float("nan")) + graph.replay() + self.assertTrue(bool(torch.isfinite(result).all())) + self.assertEqual(int(torch.count_nonzero(result)), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py b/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py new file mode 100644 index 0000000000..3c05012742 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py @@ -0,0 +1,185 @@ +"""W4A8 routing must pack inactive routes after live experts, including at EP1.""" + +import unittest +from unittest.mock import patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.moe import cutlass_w4a8_moe as w4a8 +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=8, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestCutlassW4A8InactiveRoutes(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + + def test_ep1_and_ep2_reorder_match_dense_experts(self): + """Real CUDA permutation/count/finalize kernels must ignore padded routes. + + Only the external CUTLASS GEMMs are replaced with dense expert GEMMs; + this checks routing, not native INT4 GEMM numerics. + """ + experts, hidden, intermediate = 2, 128, 128 + ids = torch.tensor( + [[1, 0], [0, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda" + ) + inputs = ( + torch.arange(4 * hidden, device="cuda") + .reshape(4, hidden) + .remainder(7) + .float() + - 3 + ) / 16 + inputs = inputs.to(torch.bfloat16) + eye = torch.eye(hidden, device="cuda") + dense_w13 = [ + torch.cat((eye * ((e + 1) / 4), eye * ((e + 2) / 4))) + for e in range(experts) + ] + dense_w2 = [eye * ((e + 1) / 2) for e in range(experts)] + + def dense_reference(dynamic): + scale1 = ( + inputs[(ids >= 0).any(dim=-1)].float().abs().amax() / 448.0 + if dynamic + else 1.0 + ) + activated = {} + for token in range(ids.shape[0]): + for slot in range(ids.shape[1]): + expert = int(ids[token, slot]) + if expert < 0: + continue + quantized = (inputs[token].float() / scale1).to(torch.float8_e4m3fn) + gate_up = F.linear( + quantized.float() * scale1, dense_w13[expert] + ).to(torch.bfloat16) + gate, up = gate_up.float().chunk(2) + activated[token, slot] = F.silu(gate) * up + scale2 = ( + torch.stack(list(activated.values())).abs().amax().clamp_min(1e-10) + / 448.0 + if dynamic + else 1.0 + ) + result = torch.zeros_like(inputs, dtype=torch.float32) + for (token, slot), value in activated.items(): + expert = int(ids[token, slot]) + quantized = (value / scale2).to(torch.float8_e4m3fn) + down = F.linear(quantized.float() * scale2, dense_w2[expert]).to( + torch.bfloat16 + ) + result[token] += down.float() * weights[token, slot] + return result + + w13_q = torch.zeros( + (experts, 2 * intermediate, hidden // 2), device="cuda", dtype=torch.int8 + ) + w2_q = torch.zeros( + (experts, hidden, intermediate // 2), device="cuda", dtype=torch.int8 + ) + scales = torch.ones((experts, 1), device="cuda") + strides = [ + torch.zeros((experts, 3), device="cuda", dtype=torch.int64) + for _ in range(8) + ] + offsets = torch.empty(experts + 1, device="cuda", dtype=torch.int32) + sizes1 = torch.empty((experts, 3), device="cuda", dtype=torch.int32) + sizes2 = torch.empty_like(sizes1) + + def dense_gemm( + out, x, packed, input_scale, _weight_scale, starts, sizes, *args + ): + out.fill_(float("nan")) + matrices = dense_w13 if packed is w13_q else dense_w2 + for expert in range(experts): + start, count = int(starts[expert]), int(sizes[expert, 1]) + if count: + values = x[start : start + count].float() * input_scale + out[start : start + count].copy_(F.linear(values, matrices[expert])) + + def run(value, *, dynamic, route_ids=ids, route_weights=weights): + return w4a8.cutlass_w4a8_moe( + value, + w13_q, + w2_q, + scales, + scales, + route_weights, + route_ids, + *strides, + offsets, + sizes1, + sizes2, + a1_scale=None if dynamic else torch.ones(1, device="cuda"), + a2_scale=None if dynamic else torch.ones(1, device="cuda"), + ) + + for ep_size in (1, 2): + with ( + get_parallel().override( + tp_size=ep_size, + tp_rank=0, + attn_tp_size=ep_size, + attn_tp_rank=0, + moe_ep_size=ep_size, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + ), + patch.object(w4a8, "cutlass_w4a8_moe_mm", side_effect=dense_gemm), + ): + for dynamic in (False, True): + reference = dense_reference(dynamic) + for poison in (0.0, 1e6, float("inf"), float("nan")): + with self.subTest( + ep_size=ep_size, dynamic=dynamic, poison=poison + ): + value = inputs.clone() + value[2].fill_(poison) + route_weights = weights.masked_fill(ids < 0, float("nan")) + actual = run( + value, dynamic=dynamic, route_weights=route_weights + ) + torch.testing.assert_close( + offsets, + torch.tensor( + [0, 3, 5], device="cuda", dtype=torch.int32 + ), + ) + self.assertTrue(torch.isfinite(actual).all().item()) + torch.testing.assert_close( + actual, reference.to(actual.dtype), rtol=1e-2, atol=1e-3 + ) + self.assertEqual(torch.count_nonzero(actual[2]).item(), 0) + inactive = torch.full_like(ids, -1) + actual = run( + torch.full_like(inputs, float("nan")), + dynamic=True, + route_ids=inactive, + route_weights=torch.full_like(weights, float("nan")), + ) + self.assertTrue(torch.isfinite(actual).all().item()) + self.assertEqual(torch.count_nonzero(actual).item(), 0) + torch.testing.assert_close( + ids, + torch.tensor( + [[1, 0], [0, -1], [-1, -1], [1, 0]], + device="cuda", + dtype=torch.int32, + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py b/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py index 9f46ba5c55..08414eac3b 100644 --- a/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py +++ b/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py @@ -10,7 +10,6 @@ from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute_with_scale from sglang.srt.layers.moe.moe_runner import humming as humming_runner from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig -from sglang.srt.layers.quantization import humming_utils from sglang.srt.utils import get_device_sm from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -37,6 +36,8 @@ class TestHummingFp8Dispatch(CustomTestCase): raise unittest.SkipTest("Humming FP8 dispatch requires SM90") def test_fp8_dispatch_preserves_group_scales(self): + from sglang.srt.layers.quantization import humming_utils + generator = torch.Generator(device="cuda").manual_seed(0) hidden_states = torch.randn( 5, @@ -258,5 +259,106 @@ class TestHummingFp8Dispatch(CustomTestCase): ) +class TestHummingInactiveRoutes(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + + def test_indexed_finalize_skips_poisoned_routes_without_ep(self): + from sglang.kernels.ops.moe.moe_fused_mul_sum import moe_fused_mul_sum + + ids = torch.tensor( + [[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda" + ) + # A legal leading storage record makes a negative map lookup observable + # without making the test depend on an out-of-allocation device read. + map_storage = torch.tensor([0, 0, -1], device="cuda", dtype=torch.int32) + for dtype in (torch.bfloat16, torch.float16): + for expert_map in (None, map_storage[1:]): + for poison_weights in (False, True): + with self.subTest( + dtype=dtype, + mapped=expert_map is not None, + poison_weights=poison_weights, + ): + valid = ids >= 0 + if expert_map is not None: + valid &= expert_map[ids.clamp_min(0)] >= 0 + values = ( + torch.arange(4 * 2 * 128, device="cuda") + .reshape(4, 2, 128) + .remainder(11) + .to(dtype) + ) + values[~valid] = float("nan") + route_weights = weights.clone() + if poison_weights: + route_weights[~valid] = float("nan") + expected = ( + ( + torch.where(valid.unsqueeze(-1), values.float(), 0) + * torch.where(valid, route_weights, 0).unsqueeze(-1) + ) + .sum(dim=1) + .to(dtype) + ) + actual = moe_fused_mul_sum( + values, + route_weights, + topk_ids=ids, + expert_map=expert_map, + is_ep=False, + ) + self.assertTrue(torch.isfinite(actual).all().item()) + torch.testing.assert_close(actual, expected) + self.assertEqual(torch.count_nonzero(actual[2]).item(), 0) + + def test_grouped_permutation_skips_inactive_routes_without_ep(self): + from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute, moe_unpermute + + ids = torch.tensor( + [[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda" + ) + inputs = ( + torch.arange(4 * 128, device="cuda") + .reshape(4, 128) + .remainder(7) + .to(torch.bfloat16) + ) + output = torch.full( + (ids.numel(), 128), float("nan"), device="cuda", dtype=inputs.dtype + ) + permuted, src2dst, offsets = moe_permute( + inputs, ids, num_experts=2, is_ep=False, outputs=output + ) + self.assertTrue((src2dst.view_as(ids)[ids < 0] < 0).all().item()) + torch.testing.assert_close( + offsets, torch.tensor([0, 2, 5], device="cuda", dtype=torch.int32) + ) + down = torch.full_like(permuted, float("nan")) + reference = torch.zeros_like(inputs) + for expert in range(2): + start, end = int(offsets[expert]), int(offsets[expert + 1]) + down[start:end] = permuted[start:end] * (expert + 1) + for token in range(ids.shape[0]): + for slot in range(ids.shape[1]): + expert = int(ids[token, slot]) + if expert >= 0: + reference[token] += ( + inputs[token] * (expert + 1) * weights[token, slot] + ) + actual = moe_unpermute(down, src2dst, ids, weights) + self.assertTrue(torch.isfinite(actual).all().item()) + torch.testing.assert_close(actual, reference, rtol=1e-2, atol=1e-2) + self.assertEqual(torch.count_nonzero(actual[2]).item(), 0) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py index 2eb478836d..f7d8884ace 100644 --- a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py +++ b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py @@ -1,17 +1,24 @@ import itertools import sys from types import SimpleNamespace +from unittest.mock import patch import pytest import torch from sgl_kernel.scalar_type import scalar_types from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm +from sglang.srt.layers import zero_copy_context from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import fused_marlin_moe +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config from sglang.srt.layers.quantization.marlin_utils_fp4 import ( prepare_moe_nvfp4_layer_for_marlin, ) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.srt.utils.common import is_sm80_supported, is_sm90_supported from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_marlin_utils import ( @@ -20,7 +27,7 @@ from sglang.test.test_marlin_utils import ( marlin_quantize, ) -register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=720, stage="base-b", runner_config="1-gpu-large") def _has_aot_moe_wna16_marlin_gemm() -> bool: @@ -896,5 +903,448 @@ def test_mxfp4_tp8_load_before_padding(tp_rank): ) +def _marlin_inactive_weights(experts, output_size, input_size, quant_type): + tensors = _setup_moe_weights( + min(experts, 8), output_size, input_size, quant_type, 128, False, torch.bfloat16 + ) + result = [] + for tensor in tensors: + if tensor is None: + result.append(None) + continue + tensor = tensor.repeat((experts // tensor.shape[0],) + (1,) * (tensor.ndim - 1)) + # A leading poisoned record makes negative expert indexing observable + # without reading outside an allocation during the failing regression. + storage = tensor.new_empty((experts + 1, *tensor.shape[1:])) + storage[0].fill_(float("nan") if tensor.is_floating_point() else 0) + storage[1:].copy_(tensor) + result.append(storage[1:]) + return result + + +@pytest.mark.parametrize( + "rows,experts,intermediate", + [(1, 8, 128), (4, 128, 128), (16, 8, 256), (64, 128, 128)], +) +@pytest.mark.parametrize( + "quant_type,num_bits", + [(scalar_types.uint4b8, 4), (scalar_types.uint8b128, 8), (scalar_types.uint4, 4)], +) +def test_fused_marlin_moe_inactive_routes( + rows, experts, intermediate, quant_type, num_bits +): + """EP1 padded graph rows contribute zero through Marlin and its shared caches.""" + _check_marlin_inactive_routes(rows, experts, intermediate, quant_type, num_bits) + + +def test_stock_marlin_lora_inactive_routes(): + if torch.cuda.get_device_capability()[0] < 9: + pytest.skip("Stock Marlin LoRA requires SM90 or newer") + _check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "stock_lora") + + +def test_factored_marlin_lora_inactive_routes(): + if torch.cuda.get_device_capability()[0] < 9: + pytest.skip("Experimental Marlin LoRA requires SM90 or newer") + _check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "factored_lora") + + +def _check_marlin_inactive_routes( + rows, experts, intermediate, quant_type, num_bits, runner_kind="fused" +): + torch.manual_seed(43) + hidden, topk = 256, 2 + with get_context().override_server_args( + model_path="dummy", + device="cuda", + moe_runner_backend="auto", + ep_dispatch_algorithm=None, + enable_eplb=False, + enable_waterfill=False, + ): + flags = get_flags().moe + flag_values = {name: getattr(flags, name) for name in flags.__struct_fields__} + flag_values["in_speculative_scope"] = False + with ( + flags.override(**flag_values), + get_parallel().override( + tp_size=1, + tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + attn_dp_size=1, + attn_cp_size=1, + tp_group=None, + ), + ): + initialize_moe_config() + x = torch.randn((rows, hidden), device="cuda", dtype=torch.bfloat16) + logits = torch.randn((rows, experts), device="cuda", dtype=torch.float32) + w1_ref, w1, s1, z1, g1, p1 = _marlin_inactive_weights( + experts, 2 * intermediate, hidden, quant_type + ) + w2_ref, w2, s2, z2, g2, p2 = _marlin_inactive_weights( + experts, hidden, intermediate, quant_type + ) + router = TopK( + top_k=topk, + layer_id=0, + renormalize=True, + output_format=TopKOutputFormat.STANDARD, + allow_routed_experts_capture=False, + ) + router.bind_runner_backend(MoeRunnerBackend.MARLIN) + live = torch.tensor(rows, device="cuda", dtype=torch.int32) + batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=1, + input_ids=torch.arange(rows, device="cuda"), + req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64), + seq_lens=torch.tensor([rows], device="cuda", dtype=torch.int32), + out_cache_loc=torch.arange(rows, device="cuda"), + seq_lens_sum=rows, + num_token_non_padded=live, + ) + output = torch.full_like(x, float("nan")) + finite_logits = logits.clone() + hook_weights = [] + if runner_kind != "fused": + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.moe_runner.marlin import MarlinMoeQuantInfo + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatchOutput, + ) + + config = MoeRunnerConfig( + num_experts=experts, + num_local_experts=experts, + hidden_size=hidden, + intermediate_size_per_partition=intermediate, + top_k=topk, + params_dtype=x.dtype, + runner_backend=MoeRunnerBackend.MARLIN, + inplace=False, + ) + quant_info = MarlinMoeQuantInfo( + w13_qweight=w1, + w2_qweight=w2, + w13_scales=s1, + w2_scales=s2, + w13_g_idx_sort_indices=p1, + w2_g_idx_sort_indices=p2, + weight_bits=num_bits, + w13_g_idx=g1, + w2_g_idx=g2, + w13_qzeros=z1, + w2_qzeros=z2, + ) + if runner_kind == "stock_lora": + from sglang.srt.lora.lora_moe_runner_marlin import ( + MarlinLoraRunnerCore, + ) + from sglang.srt.lora.lora_moe_runners import LoRAHooks + + core = MarlinLoraRunnerCore(config) + + def after_down(activation, projected, weights, ids): + hook_weights[:] = [weights, ids] + + hooks = LoRAHooks(after_down=after_down) + else: + from sglang.srt.lora.marlin_lora_temp.moe_runner import ( + _use_shared_outer_factorization, + fused_experts_experimental_sgl_marlin_lora, + ) + + # A zero adapter must preserve base output, including NaN-padded + # routing weights consumed by the factored rank-sum reduction. + rank = 8 + lora_info = SimpleNamespace( + lora_use_virtual_experts=True, + max_lora_rank=rank, + has_active_lora=True, + experts_shared_outer_loras=True, + gate_up_lora_a_weights=x.new_zeros((1, 1, 2 * rank, hidden)), + gate_up_lora_b_weights=x.new_zeros( + (1, experts, 2 * intermediate, rank) + ), + down_lora_a_weights=x.new_zeros( + (1, experts, rank, intermediate) + ), + down_lora_b_weights=x.new_zeros((1, 1, hidden, rank)), + token_lora_mapping=torch.zeros( + rows, device="cuda", dtype=torch.int32 + ), + ) + assert _use_shared_outer_factorization(lora_info, rows, topk) + + def run(): + with forward_context( + ForwardContext(attn_backend=None), forward_batch=batch + ): + routing = router(x, logits) + if runner_kind != "fused": + dispatch = StandardDispatchOutput(x, None, routing) + if runner_kind == "stock_lora": + return core.run_from_dispatch( + dispatch, quant_info, config, hooks + ).hidden_states + return fused_experts_experimental_sgl_marlin_lora( + dispatch, quant_info, config, lora_info + ).hidden_states + return fused_marlin_moe( + x, + w1, + w2, + s1, + s2, + logits, + routing.topk_weights, + routing.topk_ids, + g_idx1=g1, + g_idx2=g2, + sort_indices1=p1, + sort_indices2=p2, + w1_zeros=z1, + w2_zeros=z2, + num_bits=num_bits, + ) + + weights, ids = torch.topk(torch.softmax(logits, dim=-1), topk) + weights /= weights.sum(dim=-1, keepdim=True) + gate_up = torch.einsum("mh,mtih->mti", x, w1_ref[ids]) + gate, up = gate_up.chunk(2, dim=-1) + activated = torch.nn.functional.silu(gate) * up + projected = torch.einsum("mti,mthi->mth", activated, w2_ref[ids]) + reference = (projected.float() * weights[:, :, None]).sum(1).to(x.dtype) + with patch.object(zero_copy_context, "get_moe_output", return_value=output): + for _ in range(3): + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = run() + for count in (max(rows - 1, 0), rows, 0, min(rows, 2)): + logits.copy_(finite_logits) + logits[count:].fill_(float("nan")) + live.fill_(count) + actual.fill_(float("nan")) + graph.replay() + expected = reference.clone() + expected[count:].zero_() + torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.02) + assert torch.isfinite(actual).all() + assert torch.count_nonzero(actual[count:]) == 0 + if hook_weights: + weights, ids = hook_weights + assert torch.count_nonzero(weights[ids < 0]) == 0 + + +def test_moe_wna16_marlin_inactive_output_ownership(): + """An allocated output zeros inactive rows; caller-owned rows stay untouched.""" + torch.manual_seed(57) + m, e, k, n, topk, block_m = 1, 8, 256, 256, 2, 8 + _, qweight, scales, zeros, g_idx, permutation = _marlin_inactive_weights( + e, n, k, scalar_types.uint4b8 + ) + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + ids = torch.full((m, topk), -1, device="cuda", dtype=torch.int32) + routing_weights = torch.ones((m, topk), device="cuda", dtype=torch.float32) + sorted_ids, expert_ids, num_post = moe_align_block_size(ids, block_m, e) + workspace = torch.zeros( + torch.cuda.get_device_properties(x.device).multi_processor_count * 4, + device="cuda", + dtype=torch.int32, + ) + caller_output = torch.full((m * topk, n), 3.0, device="cuda", dtype=x.dtype) + original_empty = torch.empty + + def poisoned_empty(*args, **kwargs): + tensor = original_empty(*args, **kwargs) + if tensor.is_floating_point(): + tensor.fill_(float("nan")) + return tensor + + for output in (None, caller_output): + with patch.object(torch, "empty", side_effect=poisoned_empty): + result = _run_single_gemm( + moe_wna16_marlin_gemm, + x, + output, + qweight, + scales, + zeros, + g_idx, + permutation, + workspace, + sorted_ids, + expert_ids, + num_post, + routing_weights, + scalar_types.uint4b8, + block_m, + topk, + m, + n, + k, + False, + True, + True, + ) + expected = ( + torch.zeros_like(result) if output is None else torch.full_like(result, 3.0) + ) + torch.testing.assert_close(result, expected, rtol=0, atol=0) + + +def test_marlin_lora_direct_inactive_routes(): + """Direct LoRA projections overwrite inactive routes with finite zeros.""" + from sglang.srt.lora.marlin_lora_temp.direct_decode import ( + direct_decode_down_shrink, + direct_decode_gate_expand, + ) + + torch.manual_seed(51) + rows, topk, experts, intermediate, rank = 3, 6, 8, 384, 32 + ids = torch.tensor( + [[0, 1, 2, 3, 4, 5], [-1, -1, -1, -1, -1, -1], [2, -1, 4, 5, -1, 7]], + device="cuda", + dtype=torch.int32, + ) + slots = torch.zeros(rows, device="cuda", dtype=torch.int32) + shared = torch.randn((rows, 2 * rank), device="cuda", dtype=torch.bfloat16) + gate_b = ( + torch.randn( + (1, experts, 2 * intermediate, rank), device="cuda", dtype=torch.bfloat16 + ) + * 0.05 + ) + down_a = ( + torch.randn( + (1, experts, rank, intermediate), device="cuda", dtype=torch.bfloat16 + ) + * 0.05 + ) + activation = torch.randn( + (rows * topk, intermediate), device="cuda", dtype=torch.bfloat16 + ) + activation.view(rows, topk, intermediate)[ids < 0] = float("nan") + gate_output = torch.full( + (rows, topk, 2 * intermediate), + float("nan"), + device="cuda", + dtype=torch.bfloat16, + ) + down_output = torch.full( + (rows, topk, rank), float("nan"), device="cuda", dtype=torch.bfloat16 + ) + direct_decode_gate_expand(shared, gate_b, ids, slots, gate_output) + direct_decode_down_shrink(activation, down_a, ids, slots, down_output) + expected_gate = torch.zeros_like(gate_output) + expected_down = torch.zeros_like(down_output) + for token in range(rows): + for route in range(topk): + expert = int(ids[token, route]) + if expert < 0: + continue + for half in range(2): + start = half * intermediate + expected_gate[token, route, start : start + intermediate] = ( + gate_b[0, expert, start : start + intermediate].float() + @ shared[token, half * rank : (half + 1) * rank].float() + ).to(gate_output.dtype) + expected_down[token, route] = ( + down_a[0, expert].float() @ activation[token * topk + route].float() + ).to(down_output.dtype) + torch.testing.assert_close(gate_output, expected_gate, rtol=0.03, atol=0.01) + torch.testing.assert_close(down_output, expected_down, rtol=0.03, atol=0.01) + assert torch.count_nonzero(gate_output[ids < 0]) == 0 + assert torch.count_nonzero(down_output[ids < 0]) == 0 + + +@pytest.mark.parametrize("fused_norm", [False, True]) +def test_muse_marlin_inactive_routes(fused_norm): + """Muse maps global routes once and masks inactive weights before either norm tail.""" + from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts + + torch.manual_seed(67) + rows, local_experts, hidden, intermediate, topk = 4, 2, 256, 256, 2 + dtype = torch.bfloat16 + layer = MuseSpark1xFp4Experts.__new__(MuseSpark1xFp4Experts) + torch.nn.Module.__init__(layer) + layer.quant_config = SimpleNamespace(group_size=16) + layer.moe_runner_config = SimpleNamespace(is_gated=True) + layer.params_dtype = dtype + layer.intermediate_size_per_partition = intermediate + layer.num_local_experts = local_experts + layer.global_num_experts = 4 + layer._expert_map = torch.tensor([-1, -1, 0, 1], device="cuda", dtype=torch.int32) + references = [] + for prefix, n, k in ( + ("w13", 2 * intermediate, hidden), + ("w2", hidden, intermediate), + ): + packed, scales, global_scales, refs = zip( + *[make_nvfp4_weight_and_ref(n, k, dtype) for _ in range(local_experts)] + ) + for suffix, tensors in ( + ("weight", packed), + ("weight_scale", scales), + ("weight_scale_2", global_scales), + ): + setattr( + layer, + f"{prefix}_{suffix}", + torch.nn.Parameter(torch.stack(tensors), requires_grad=False), + ) + references.append(torch.stack(refs)) + prepare_moe_nvfp4_layer_for_marlin(layer) + x = torch.randn((rows, hidden), device="cuda", dtype=dtype) / 20 + ids = torch.tensor( + [[2, 3], [3, -1], [0, 2], [-1, -1]], device="cuda", dtype=torch.int32 + ) + local_ids = torch.tensor( + [[0, 1], [1, -1], [-1, 0], [-1, -1]], device="cuda", dtype=torch.int32 + ) + weights = torch.tensor( + [[0.25, 0.75], [0.5, 0.5], [0.25, 0.75], [0.5, 0.5]], device="cuda" + ) + weights[local_ids < 0] = float("nan") + norm = torch.nn.RMSNorm(hidden, eps=1e-6, device="cuda", dtype=dtype) + layer._post_expert_norm = norm + layer._post_expert_norm_weight = norm.weight.detach() if fused_norm else None + layer._post_expert_norm_eps = norm.eps + layer._post_expert_norm_weight_first = False + + expert_output, mapped = layer._marlin_expert_outputs(x, weights, ids) + torch.testing.assert_close(mapped, local_ids, rtol=0, atol=0) + assert torch.count_nonzero(expert_output[local_ids < 0]) == 0 + expected_experts = torch.zeros((rows, topk, hidden), device="cuda", dtype=dtype) + w13_ref, w2_ref = references + for row in range(rows): + for route in range(topk): + expert = int(local_ids[row, route]) + if expert < 0: + continue + gate_up = x[row] @ w13_ref[expert].T + gate, up = gate_up.chunk(2) + activated = torch.nn.functional.silu(gate) * up + expected_experts[row, route] = activated @ w2_ref[expert].T + torch.testing.assert_close(expert_output, expected_experts, rtol=0.05, atol=0.25) + safe_weights = weights.masked_fill(local_ids < 0, 0.0) + expected = ( + (norm(expected_experts).float() * safe_weights[..., None]).sum(1).to(dtype) + ) + actual = layer._forward_marlin(x, weights, ids, norm) + torch.testing.assert_close(actual, expected, rtol=0.05, atol=0.25) + assert torch.isfinite(actual).all() + assert torch.count_nonzero(actual[-1]) == 0 + assert torch.isnan(weights[local_ids < 0]).all() + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/moe/test_shared_expert_routing_extra.py b/test/registered/moe/test_shared_expert_routing_extra.py index 13032aa60b..eb6cab213c 100644 --- a/test/registered/moe/test_shared_expert_routing_extra.py +++ b/test/registered/moe/test_shared_expert_routing_extra.py @@ -59,7 +59,26 @@ class TestSelectExpertsPerRankSharedSlots(CustomTestCase): "has_per_rank_fused_shared_slots", side_effect=lambda num_fused: num_fused > 0, ), - get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), + # Model one rank of TP=EP; routing itself issues no collectives. + get_parallel().override( + tp_size=self.EP_SIZE, + tp_rank=self.EP_RANK, + tp_group=None, + attn_tp_size=self.EP_SIZE, + attn_tp_rank=self.EP_RANK, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_ep_size=self.EP_SIZE, + moe_ep_rank=self.EP_RANK, + moe_ep_group=None, + moe_tp_size=1, + moe_tp_rank=0, + moe_dp_size=1, + moe_dp_rank=0, + ), ): out = select_experts( torch.empty(logits.shape[0], 16, device=logits.device), diff --git a/test/registered/moe/test_triton_moe_padding.py b/test/registered/moe/test_triton_moe_padding.py new file mode 100644 index 0000000000..134592b7d9 --- /dev/null +++ b/test/registered/moe/test_triton_moe_padding.py @@ -0,0 +1,309 @@ +"""Padded routes must contribute zero even when every expert is local.""" + +import unittest + +import torch +import torch.nn.functional as F +import triton.language as tl + +from sglang.kernels.ops.moe.fused_moe_triton_kernels import invoke_fused_moe_kernel +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import ( + TritonMoeQuantInfo, + TritonRunnerCore, + pre_permute_standard_to_triton, +) +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small") + + +class TestTritonMoePadding(CustomTestCase): + ROWS, EXPERTS, TOP_K, HIDDEN, INTERMEDIATE = 4, 4, 2, 128, 128 + + def setUp(self): + super().setUp() + self.enterContext( + get_context().override_server_args( + model_path="dummy", + device="cuda", + moe_runner_backend="triton", + ep_dispatch_algorithm=None, + enable_eplb=False, + enable_waterfill=False, + ) + ) + moe_flags = get_flags().moe + scoped_flags = { + name: getattr(moe_flags, name) for name in moe_flags.__struct_fields__ + } + scoped_flags["in_speculative_scope"] = False + self.enterContext(moe_flags.override(**scoped_flags)) + initialize_moe_config() + self.enterContext( + get_parallel().override( + tp_size=1, + tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + attn_dp_size=1, + attn_cp_size=1, + tp_group=None, + ) + ) + torch.manual_seed(123) + + @staticmethod + def _expert_view(shape, dtype, *, quantized=False): + # A normal view with an extra leading record makes selecting an inactive + # expert numerically observable. Only actual experts contain finite data. + if quantized: + storage = torch.randint( + 120, 137, (shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype + ) + else: + storage = ( + torch.randn((shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype) + * 0.05 + ) + storage[0].fill_(float("nan")) + return storage[1:] + + def _fixture(self, dtype, quantized=False): + x = torch.randn((self.ROWS, self.HIDDEN), device="cuda", dtype=dtype) * 0.1 + logits = torch.tensor( + [ + [9.0, 8.0, 1.0, 0.0], + [0.0, 9.0, 8.0, 1.0], + [1.0, 0.0, 9.0, 8.0], + [8.0, 1.0, 0.0, 9.0], + ], + device="cuda", + ) + shapes = [ + (self.EXPERTS, 2 * self.INTERMEDIATE, self.HIDDEN), + (self.EXPERTS, self.HIDDEN, self.INTERMEDIATE), + ] + if quantized: + weights = [ + self._expert_view(shape, torch.uint8, quantized=True) + for shape in shapes + ] + scales = [] + for shape in shapes: + storage = torch.full( + (self.EXPERTS + 1, shape[1], shape[2] // 32), + 0.01, + device="cuda", + dtype=dtype, + ) + storage[0].fill_(float("nan")) + scales.append(storage[1:]) + quant = TritonMoeQuantInfo( + *weights, + use_int8_w8a16=True, + w13_scale=scales[0], + w2_scale=scales[1], + block_shape=[0, 32], + ) + dense_weights = [ + (weight.to(dtype) - 128) * scale.repeat_interleave(32, dim=-1) + for weight, scale in zip(weights, scales) + ] + else: + weights = [self._expert_view(shape, dtype) for shape in shapes] + quant = TritonMoeQuantInfo(*weights) + dense_weights = weights + router = TopK( + top_k=self.TOP_K, + layer_id=0, + output_format=TopKOutputFormat.STANDARD, + allow_routed_experts_capture=False, + ) + router.bind_runner_backend(MoeRunnerBackend.TRITON) + batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=self.ROWS, + input_ids=torch.arange(self.ROWS, device="cuda"), + req_pool_indices=torch.arange(self.ROWS, device="cuda"), + seq_lens=torch.ones(self.ROWS, device="cuda", dtype=torch.int32), + out_cache_loc=torch.arange(self.ROWS, device="cuda"), + seq_lens_sum=self.ROWS, + num_token_non_padded=torch.tensor(2, device="cuda", dtype=torch.int32), + ) + config = MoeRunnerConfig( + num_experts=self.EXPERTS, + num_local_experts=self.EXPERTS, + hidden_size=self.HIDDEN, + intermediate_size_per_partition=self.INTERMEDIATE, + top_k=self.TOP_K, + num_fused_shared_experts=0, + params_dtype=dtype, + runner_backend=MoeRunnerBackend.TRITON, + inplace=False, + ) + return x, logits, quant, dense_weights, router, batch, config + + @staticmethod + def _prepare(x, routes, quant, config): + state = {} + prepared = pre_permute_standard_to_triton( + StandardDispatchOutput(x, None, routes), quant, config, state + ) + return prepared, state + + def _run(self, x, routes, quant, config): + prepared, state = self._prepare(x, routes, quant, config) + return TritonRunnerCore(config).run(prepared, quant, state).hidden_states + + def _dense_reference(self, x, logits, weights): + probabilities = torch.softmax(logits, dim=-1) + scores, ids = probabilities.topk(self.TOP_K, dim=-1) + scores /= scores.sum(dim=-1, keepdim=True) + result = torch.zeros_like(x) + for row in range(x.shape[0]): + for slot in range(self.TOP_K): + expert = int(ids[row, slot]) + gate, up = F.linear(x[row], weights[0][expert]).chunk(2) + activated = (F.silu(gate.float()) * up.float()).to(x.dtype) + result[row] += ( + F.linear(activated, weights[1][expert]).float() * scores[row, slot] + ).to(x.dtype) + return result + + def test_runner_observes_updated_batch_bound(self): + """An inactive row must stay finite and zero through both expert GEMMs.""" + for dtype, quantized in [ + (torch.bfloat16, False), + (torch.float16, False), + (torch.bfloat16, True), + ]: + with self.subTest(dtype=dtype, quantized=quantized): + x, logits, quant, weights, router, batch, config = self._fixture( + dtype, quantized + ) + dense = self._dense_reference(x, logits, weights) + # Reuse the batch and its device scalar, as static graph buffers do. + with forward_context( + ForwardContext(attn_backend=None), forward_batch=batch + ): + for live in (2, 3, 0, self.ROWS): + with self.subTest(live=live): + batch.num_token_non_padded.fill_(live) + routes = router.forward_cuda(x, logits) + self.assertTrue(torch.all(routes.topk_ids[live:] == -1)) + output = self._run(x, routes, quant, config) + self.assertTrue(torch.isfinite(output).all()) + torch.testing.assert_close( + output[live:], + torch.zeros_like(output[live:]), + rtol=0, + atol=0, + ) + torch.testing.assert_close( + output[:live], dense[:live], rtol=0.04, atol=2e-4 + ) + if live: + # A cropped input has no inactive routes at all. + cropped_routes = router.forward_cuda( + x[:live], logits[:live] + ) + cropped = self._run( + x[:live], cropped_routes, quant, config + ) + torch.testing.assert_close( + output[:live], cropped, rtol=0.01, atol=2e-5 + ) + + def test_inactive_routes_preserve_output_layouts(self): + """Skipping a route must respect sorted, accumulated and half-width outputs.""" + x, logits, quant, _, router, batch, config = self._fixture(torch.bfloat16) + with forward_context(ForwardContext(attn_backend=None), forward_batch=batch): + routes = router.forward_cuda(x, logits) + prepared, state = self._prepare(x, routes, quant, config) + count = int(prepared.num_tokens_post_padded) + block = state["config"]["BLOCK_SIZE_M"] + sorted_ids = prepared.sorted_token_ids[:count].tolist() + expert_ids = prepared.expert_ids[: (count + block - 1) // block].tolist() + for layout in ("sorted", "add", "sum", "swiglu"): + with self.subTest(layout=layout): + width = ( + self.INTERMEDIATE if layout == "swiglu" else 2 * self.INTERMEDIATE + ) + rows = ( + count + if layout == "sorted" + else self.ROWS + if layout == "sum" + else self.ROWS * self.TOP_K + ) + output = torch.full((rows, width), 0.375, device="cuda", dtype=x.dtype) + expected = output.clone() + for position, route in enumerate(sorted_ids): + expert = expert_ids[position // block] + if expert == -1 or route >= self.ROWS * self.TOP_K: + continue + value = F.linear(x[route // self.TOP_K], quant.w13_weight[expert]) + if layout == "swiglu": + gate, up = value[::2], value[1::2] + value = (F.silu(gate.float()) * up.float()).to(x.dtype) + target = ( + position + if layout == "sorted" + else route // self.TOP_K + if layout == "sum" + else route + ) + if layout in ("add", "sum"): + expected[target] += value + else: + expected[target] = value + invoke_fused_moe_kernel( + x, + quant.w13_weight, + None, + output, + None, + None, + None, + routes.topk_weights, + routes.topk_ids, + prepared.sorted_token_ids, + prepared.expert_ids, + prepared.num_tokens_post_padded, + False, + self.TOP_K, + state["config"], + compute_type=tl.bfloat16, + use_fp8_w8a8=False, + use_int8_w8a8=False, + use_int8_w8a16=False, + use_int4_w4a16=False, + per_channel_quant=False, + filter_expert=False, + c_sorted=layout == "sorted", + fuse_add_to_output=layout == "add", + add_output_mask=torch.ones( + self.ROWS, device="cuda", dtype=torch.bool + ), + fuse_sum_all_reduce=layout == "sum", + router_topk=self.TOP_K, + fuse_swiglu=layout == "swiglu", + ) + self.assertTrue(torch.isfinite(output).all()) + torch.testing.assert_close(output, expected, rtol=0.02, atol=2e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py b/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py index adaa18d8a2..b900638a68 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py +++ b/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py @@ -4,6 +4,7 @@ from unittest.mock import patch import torch +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferDispatcher, _WorkspaceNamespace, @@ -126,10 +127,20 @@ class TestFlashinferDispatcher(CustomTestCase): router_logits = torch.tensor( [[0.1, 3.0, 0.2, 2.0], [5.0, 0.0, 4.0, 0.1], [0.0, 0.1, 0.2, 9.0]] ) + logical_to_physical = torch.tensor([2, 0, 3, 1]) topk_output = BypassedTopKOutput( hidden_states=hidden_states, - router_logits=router_logits, + raw_router_logits=router_logits, topk_config=TopKConfig(top_k=2, torch_native=True), + expert_location_dispatch_info=ExpertLocationDispatchInfo( + ep_dispatch_algorithm="static", + partial_logical_to_rank_dispatch_physical_map=logical_to_physical, + partial_logical_to_all_physical_map=logical_to_physical[:, None], + partial_logical_to_all_physical_map_num_valid=torch.ones( + 4, dtype=torch.int64 + ), + num_physical_experts=4, + ), ) _dispatch_without_dp( @@ -142,7 +153,7 @@ class TestFlashinferDispatcher(CustomTestCase): expert_ids = moe_a2a.token_selected_experts self.assertEqual(expert_ids.dtype, torch.int32) self.assertEqual( - expert_ids.sort(dim=-1).values.tolist(), [[1, 3], [0, 2], [2, 3]] + expert_ids.sort(dim=-1).values.tolist(), [[0, 1], [2, 3], [1, 3]] ) topk_weights = moe_a2a.payloads[moe_a2a.expert_id_payload_index + 1] self.assertEqual(topk_weights.shape, (3, 2)) diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index a90c424a9d..47a04ea7d3 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -531,13 +531,13 @@ class TestFusedSharedExpertScaling(CustomTestCase): def test_glm53_idle_rank_preserves_routed_topk_contract(self): """An idle rank must return the pre-append top-8 shape; returning top-9 here would make the AITER postprocessor append shared ID 288 twice.""" - topk = TopK.__new__(TopK) - torch.nn.Module.__init__(topk) - topk.topk_config = TopKConfig( - top_k=9, shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) - ) - topk.enable_waterfill = False - topk.waterfill_balancer = None + with ( + get_context().override_server_args(enable_waterfill=False), + get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ), + ): + topk = TopK(top_k=8, layer_id=0) with ( patch.object( diff --git a/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py index 95643d3197..b1b865e54e 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py +++ b/test/registered/unit/layers/moe/test_mega_moe_decode_capacity.py @@ -8,6 +8,7 @@ collectives and graph replay are out of scope. import sys from types import ModuleType, SimpleNamespace +from unittest.mock import Mock import pytest import torch @@ -61,11 +62,6 @@ def runtime(monkeypatch): monkeypatch.setattr(mega, "get_dp_global_num_tokens", lambda: state.global_tokens) monkeypatch.setattr(mega, "_ensure_mori_shmem", lambda: None) monkeypatch.setattr(mega, "_ep_rank_world", lambda: (state.rank, 8)) - monkeypatch.setattr( - mega, - "ExpertLocationDispatchInfo", - SimpleNamespace(init_new=lambda layer_id: None), - ) utils = ModuleType("sglang.srt.models.deepseek_common.utils") utils._use_aiter = True monkeypatch.setitem(sys.modules, utils.__name__, utils) @@ -100,6 +96,13 @@ def runtime(monkeypatch): state.extend = extend state.capture = capture state.global_tokens = global_tokens + topk = Mock( + return_value=SimpleNamespace( + topk_ids=torch.zeros(local_rows, TOPK, dtype=torch.int64), + topk_weights=torch.ones(local_rows, TOPK), + ) + ) + topk.routing_inputs.return_value = (None, None) moe = SimpleNamespace( config=SimpleNamespace( hidden_size=HIDDEN, @@ -110,10 +113,7 @@ def runtime(monkeypatch): num_fused_shared_experts=0, experts=state.layer if layer is None else layer, gate=lambda h, forward_batch=None: torch.zeros(h.shape[0], EXPERTS), - topk=lambda h, logits, **kwargs: SimpleNamespace( - topk_ids=torch.zeros(h.shape[0], TOPK, dtype=torch.int64), - topk_weights=torch.ones(h.shape[0], TOPK), - ), + topk=topk, is_hash=False, layer_id=0, routed_scaling_factor=1.0, diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 0b33642463..0dfc57247c 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -17,10 +17,17 @@ maybe_stub_sgl_kernel() from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe from sglang.srt.layers.moe import topk as topk_module from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module -from sglang.srt.layers.moe.topk import TopKConfig +from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import draft_model_build_scope from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod -from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context +from sglang.srt.runtime_context import ( + get_context, + get_exec, + get_flags, + get_model, + get_parallel, +) register_cpu_ci(est_time=8, suite="base-a-test-cpu") @@ -257,11 +264,6 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): ), # Toy shapes. patch.object(mega_moe, "check_mega_moe_shapes"), - patch.object( - mega_moe.ExpertLocationDispatchInfo, - "init_new", - return_value=object(), - ), get_parallel().override( moe_ep_group=SimpleNamespace(device_group=object()) ), @@ -356,6 +358,14 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): expert boundary exposes accidentally routing image rows as text, and padded rows must contribute nothing. """ + config = get_context().override_server_args( + model_path="dummy", + enable_waterfill=False, + ep_dispatch_algorithm=None, + enable_eplb=False, + ) + config.install() + self.addCleanup(config.restore) for fused_shared in (0, 1): for scale_in_topk in (False, True): with self.subTest( @@ -383,12 +393,17 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): with ( get_flags().moe.override(a2a_backend=MoeA2ABackend.MEGAMOE), - get_parallel().override(moe_ep_size=2, moe_ep_rank=1), - patch.object( - mega_moe.ExpertLocationDispatchInfo, - "init_new", - return_value=None, + get_parallel().override( + tp_size=2, + tp_rank=1, + attn_tp_size=2, + attn_tp_rank=1, + moe_ep_size=2, + moe_ep_rank=1, + moe_tp_size=1, + moe_tp_rank=0, ), + forward_context(ForwardContext(None), forward_batch=batch), patch.object(mega_moe, "run_mega_routed_experts", run_experts), # The fused padded-row fill requires Triton/GPU. patch( @@ -437,42 +452,46 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): @staticmethod def _v41_moe(fused_shared, scale_in_topk): - return SimpleNamespace( - config=SimpleNamespace( - hidden_size=4, - num_experts_per_tok=2, - moe_intermediate_size=8, - image_token_id=99, + with ( + get_context().override_server_args(), + get_model().override( + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=fused_shared + ) ), - experts=SimpleNamespace( - should_fuse_routed_scaling_factor_in_topk=scale_in_topk, - moe_runner_config=SimpleNamespace(swiglu_limit=None), - uses_expert_location=False, - ), - gate=MagicMock( - return_value=torch.zeros((3, 4)), - e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]), - e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]), - ), - topk=SimpleNamespace( - topk_config=TopKConfig( - top_k=2 + fused_shared, - shared_experts=SharedExpertsConfig( - num_fused_shared_experts=fused_shared - ), + ): + return SimpleNamespace( + config=SimpleNamespace( + hidden_size=4, + num_experts_per_tok=2, + moe_intermediate_size=8, + image_token_id=99, + ), + experts=SimpleNamespace( + should_fuse_routed_scaling_factor_in_topk=scale_in_topk, + moe_runner_config=SimpleNamespace(swiglu_limit=None), + ), + gate=MagicMock( + return_value=torch.zeros((3, 4)), + e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]), + e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]), + ), + topk=TopK( + top_k=2, + layer_id=0, + scoring_func="sqrtsoftplus", renormalize=True, routed_scaling_factor=2.0, apply_routed_scaling_factor_on_output=scale_in_topk, - ) - ), - is_hash=False, - num_fused_shared_experts=fused_shared, - layer_id=0, - routed_scaling_factor=2.0, - routed_scaling_factor_fused_in_topk=scale_in_topk, - mega_shared_l1_weights=None, - mega_shared_l2_weights=None, - ) + ), + is_hash=False, + num_fused_shared_experts=fused_shared, + layer_id=0, + routed_scaling_factor=2.0, + routed_scaling_factor_fused_in_topk=scale_in_topk, + mega_shared_l1_weights=None, + mega_shared_l2_weights=None, + ) def test_shape_check_uses_pinned_fp8_fp4_alignment(self): # Packed SF groups need 128 elements, not a 16-byte token SF row. diff --git a/test/registered/unit/layers/moe/test_topk_routing_inputs.py b/test/registered/unit/layers/moe/test_topk_routing_inputs.py new file mode 100644 index 0000000000..a051776a25 --- /dev/null +++ b/test/registered/unit/layers/moe/test_topk_routing_inputs.py @@ -0,0 +1,724 @@ +"""Routing follows the owning layer and active batch across target/draft and TBO.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.batch_overlap.operations import ( + ExecutionOperation, + _StageExecutor, +) +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.environ import envs +from sglang.srt.eplb import expert_location_dispatch +from sglang.srt.eplb.expert_location import ExpertLocationMetadata +from sglang.srt.hardware_backend.npu.moe.topk import fused_topk_npu +from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.layers.moe import hash_topk as hash_topk_module +from sglang.srt.layers.moe import topk as topk_module +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.hash_topk import HashTopK +from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher +from sglang.srt.layers.moe.topk import ( + StandardTopKOutputPacked, + TopK, + TopKOutputFormat, + fused_topk_torch_native, +) +from sglang.srt.layers.moe.utils import ( + MoeRunnerBackend, + draft_model_build_scope, + initialize_moe_config, +) +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + get_forward_context, + has_forward_context, +) +from sglang.srt.multimodal.dsv41.vl_routing import vision_topk +from sglang.srt.runtime_context import ( + get_context, + get_exec, + get_model, + get_parallel, + get_resources, +) +from sglang.srt.speculative.eagle_info import EagleDraftInput +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def _batch(real): + empty = torch.empty(0, dtype=torch.int32) + batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=4, + input_ids=torch.arange(4), + req_pool_indices=empty, + seq_lens=empty, + out_cache_loc=empty, + seq_lens_sum=0, + ) + batch.num_token_non_padded = torch.tensor(real, dtype=torch.int32) + batch.global_num_token_non_padded = batch.num_token_non_padded + batch.dp_padding_mode = DpPaddingMode.SUM_LEN + batch.global_dp_buffer_len = batch.tbo_padded_len = 4 + return batch + + +def _placement(num_layers=1): + logical_to_physical = torch.tensor([[2, 0, 3, 1]], dtype=torch.int64).repeat( + num_layers, 1 + ) + physical_to_logical = logical_to_physical.argsort(dim=-1) + all_physical = torch.full((num_layers, 4, 4), -1, dtype=torch.int64) + all_physical[:, :, 0] = logical_to_physical + return ExpertLocationMetadata( + physical_to_logical_map=physical_to_logical, + physical_to_logical_map_cpu=physical_to_logical.clone(), + logical_to_all_physical_map=all_physical, + logical_to_all_physical_map_cpu=all_physical.clone(), + logical_to_all_physical_map_num_valid=torch.ones( + (num_layers, 4), dtype=torch.int64 + ), + ep_size=1, + logical_to_rank_dispatch_physical_map=logical_to_physical, + ) + + +class TestTopKRoutingInputs(CustomTestCase): + def setUp(self): + super().setUp() + config = get_context().override_server_args( + ep_dispatch_algorithm="static", enable_eplb=True + ) + config.install() + self.addCleanup(config.restore) + initialize_moe_config() + topology = get_parallel().override( + attn_dp_size=1, attn_cp_size=1, attn_tp_size=1, attn_tp_rank=0 + ) + topology.__enter__() + self.addCleanup(topology.__exit__, None, None, None) + for name in ("_is_cuda", "_is_hip", "_use_aiter"): + scope = patch.object(topk_module, name, False) + scope.start() + self.addCleanup(scope.stop) + self.metadata = _placement() + get_resources().expert_location_metadata = self.metadata + self.addCleanup(setattr, get_resources(), "expert_location_metadata", None) + self.hidden = torch.ones((4, 2)) + self.logits = torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(4, 1) + + def _router(self, **kwargs): + shared_slots = kwargs.pop("num_fused_shared_experts", 0) + layer_id = kwargs.pop("layer_id", 0) + with get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=shared_slots) + ): + return TopK( + top_k=2 - shared_slots, + layer_id=layer_id, + custom_routing_function=fused_topk_torch_native, + **kwargs, + ) + + def test_target_and_draft_remap_only_the_target_and_keep_mask(self): + """Draft routes must not use the target placement or lose every live row.""" + target = self._router() + with draft_model_build_scope(): + draft = self._router() + batch = _batch(2) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=batch + ): + target_ids = target.forward_native(self.hidden, self.logits).topk_ids + draft_ids = draft.forward_native(self.hidden, self.logits).topk_ids + self.assertEqual(target_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]]) + self.assertEqual(draft_ids.tolist(), [[0, 1], [0, 1], [-1, -1], [-1, -1]]) + self.assertFalse(draft.topk_config.allow_routed_experts_capture) + self.assertTrue(target.topk_config.allow_routed_experts_capture) + + def test_late_preparation_and_in_place_rebalance(self): + """Forward preparation and rebalance must be observed after scope entry.""" + router = self._router() + batch = _batch(1) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=batch + ): + before = router.forward_native(self.hidden, self.logits).topk_ids + batch.num_token_non_padded = torch.tensor(3) + self.metadata.logical_to_rank_dispatch_physical_map.copy_( + torch.tensor([[1, 3, 0, 2]]) + ) + after = router.forward_native(self.hidden, self.logits).topk_ids + self.assertEqual(before[0].tolist(), [2, 0]) + self.assertEqual(before[1].tolist(), [-1, -1]) + self.assertEqual(after.tolist(), [[1, 3], [1, 3], [1, 3], [-1, -1]]) + + def test_nested_scope_restores_and_retains_replay_buffer_identity(self): + router = self._router() + parent, child = _batch(3), _batch(1) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=parent + ): + original = get_forward_context() + bound, _ = router.routing_inputs() + self.assertIs(bound, parent.num_token_non_padded) + with self.assertRaisesRegex(RuntimeError, "child failed"): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=child + ): + self.assertIs( + router.routing_inputs()[0], child.num_token_non_padded + ) + raise RuntimeError("child failed") + self.assertIs(get_forward_context(), original) + parent.num_token_non_padded.fill_(2) + self.assertIs(router.routing_inputs()[0], bound) + self.assertEqual( + router.forward_native(self.hidden, self.logits).topk_ids[-2:].tolist(), + [[-1, -1], [-1, -1]], + ) + self.assertFalse(has_forward_context()) + + def test_tbo_children_bind_bounds_without_tbo_attention_backend(self): + router = self._router() + seen = [] + + def read_bound(state, forward_batch): + seen.append(int(router.routing_inputs()[0])) + return {"forward_batch": forward_batch} + + stages = [[ExecutionOperation("route", read_bound)]] * 2 + parent, child_a, child_b = _batch(4), _batch(1), _batch(3) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=parent + ): + a = _StageExecutor("a", stages, {"forward_batch": child_a}) + b = _StageExecutor("b", stages, {"forward_batch": child_b}) + a.next() + b.next() + a.next() + b.next() + self.assertEqual(int(router.routing_inputs()[0]), 4) + self.assertEqual(seen, [1, 3, 1, 3]) + + def test_hash_router_uses_same_placement_and_bound(self): + router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0) + with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(2) + ): + output = router(self.hidden, self.logits, torch.arange(4)) + self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [-1, -1], [-1, -1]]) + + def test_vision_router_maps_routed_columns_and_masks_shared_rows_last(self): + router = self._router( + num_fused_shared_experts=1, + routed_scaling_factor=2.0, + fused_shared_experts_scaling_factor=0.25, + ) + moe = SimpleNamespace( + topk=router, + layer_id=0, + config=SimpleNamespace(image_token_id=99), + gate=SimpleNamespace( + e_score_correction_bias=torch.zeros(4), + e_score_correction_bias_vl=torch.zeros(4), + ), + ) + with patch( + "sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False + ): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(2) + ): + output = vision_topk(moe, self.logits, torch.tensor([1, 99, 1, 99])) + self.assertEqual(output.topk_ids.tolist(), [[2, 4], [2, 4], [-1, -1], [-1, -1]]) + torch.testing.assert_close(output.topk_weights[:2, -1], torch.full((2,), 0.125)) + torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2))) + + def test_vision_packed_output_follows_built_runner(self): + """A requested MXFP4 fallback must not emit an unused packed-ID buffer.""" + with get_context().override_server_args(moe_runner_backend="flashinfer_mxfp4"): + initialize_moe_config() + for backend in ( + None, + MoeRunnerBackend.TRITON, + MoeRunnerBackend.FLASHINFER_MXFP4, + ): + with self.subTest(backend=backend): + router = self._router(is_draft=True) + router.bind_runner_backend(backend) + moe = SimpleNamespace( + topk=router, + layer_id=0, + config=SimpleNamespace(image_token_id=99), + gate=SimpleNamespace( + e_score_correction_bias=torch.zeros(4), + e_score_correction_bias_vl=torch.zeros(4), + ), + ) + with ( + patch( + "sglang.srt.multimodal.dsv41.vl_routing.is_cuda", + return_value=True, + ), + patch( + "sglang.kernels.ops.moe.moe_fused_gate.moe_fused_gate", + return_value=( + torch.full((4, 2), 0.5), + torch.tensor([[0, 1]], dtype=torch.int32).repeat(4, 1), + ), + ) as gate, + ): + output = vision_topk(moe, self.logits, input_ids=None) + packed = gate.call_args.kwargs["packed_out"] + expected_packed = backend == MoeRunnerBackend.FLASHINFER_MXFP4 + self.assertEqual(packed is not None, expected_packed) + self.assertEqual( + isinstance(output, StandardTopKOutputPacked), expected_packed + ) + + def test_vision_fake_algorithm_neutralizes_both_biases(self): + with get_exec().moe.override(ep_dispatch_algorithm="fake"): + router = self._router() + moe = SimpleNamespace( + topk=router, + layer_id=0, + config=SimpleNamespace(image_token_id=99), + gate=SimpleNamespace( + e_score_correction_bias=torch.tensor([0.0, 0.0, 1000.0, 0.0]), + e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 0.0, 1000.0]), + ), + ) + with patch( + "sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False + ): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(4) + ): + logits = self.logits.clone() + output = vision_topk(moe, logits, torch.tensor([1, 99, 1, 99])) + logical = logits.topk(2, dim=-1).indices + expected = self.metadata.logical_to_all_physical_map[0, logical, 0] + self.assertTrue(torch.equal(output.topk_ids.long(), expected)) + + def test_draft_without_input_ids_retains_prepared_live_width(self): + """Deferred draft IDs must not turn a live draft's padding bound into zero.""" + batch = _batch(0) + batch.input_ids = None + batch.spec_info = EagleDraftInput(num_tokens_per_req=2) + with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4): + batch._init_num_token_non_padded(device="cpu", is_draft_worker=True) + self.assertEqual(batch.global_num_token_non_padded_cpu, 8) + self.assertEqual(int(batch.global_num_token_non_padded), 8) + batch.forward_mode = ForwardMode.IDLE + with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4): + batch._init_num_token_non_padded(device="cpu", is_draft_worker=True) + self.assertEqual(int(batch.global_num_token_non_padded), 0) + + def test_npu_routes_keep_valid_ids_and_zero_only_padded_contributions(self): + """NPU padding must stay inert without inventing invalid or duplicate routes.""" + router = self._router(num_fused_shared_experts=1) + router.topk_config.scoring_func = "sqrtsoftplus" + logical = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32) + weights = torch.tensor([[0.75, 0.25]]).repeat(4, 1) + + class Capture: + def capture(self, *, layer_id, topk_indices): + self.ids = topk_indices.clone() + + capture = Capture() + get_resources().experts_capturer = capture + self.addCleanup(setattr, get_resources(), "experts_capturer", None) + _, info = router.routing_inputs() + with patch.object( + torch.ops.custom, + "npu_moe_gating_top_k", + create=True, + return_value=(weights, logical, None), + ): + output = fused_topk_npu( + hidden_states=self.hidden, + router_logits=self.logits, + topk_config=router.topk_config, + num_token_non_padded=torch.tensor(2), + expert_location_dispatch_info=info, + layer_id=0, + ) + self.assertEqual(output.topk_ids.tolist(), [[2, 4], [0, 4], [3, 4], [1, 4]]) + torch.testing.assert_close( + output.topk_weights[:2], torch.tensor([[0.75, 0.25]]).repeat(2, 1) + ) + torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2))) + self.assertEqual(capture.ids.tolist(), [[0], [1], [2], [3]]) + + def test_in_kernel_formats_do_not_silently_ignore_expert_placement(self): + router = self._router(output_format=TopKOutputFormat.TRITON_KERNEL) + with self.assertRaisesRegex( + NotImplementedError, "placement remapping requires" + ): + router.forward_cuda(self.hidden, self.logits) + + router = self._router(output_format=TopKOutputFormat.BYPASSED) + output = router.forward_cuda(self.hidden, self.logits) + with self.assertRaisesRegex( + NotImplementedError, "placement remapping requires" + ): + _ = output.router_logits + + def test_bypassed_materialization_preserves_placement_and_padding(self): + """A dispatcher can remap lazy routes before sending physical expert IDs.""" + router = self._router(output_format=TopKOutputFormat.BYPASSED) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(2) + ): + output = router.forward_cuda(self.hidden, self.logits) + materialized = output.to_standard(layer_id=0) + self.assertEqual( + materialized.topk_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]] + ) + self.assertIs(materialized.router_logits, self.logits) + layer = SimpleNamespace( + _use_ascend_fuseep=False, + forward_impl=lambda hidden, routes, **kwargs: routes.to_standard( + layer_id=0 + ), + ) + with patch( + "sglang.srt.layers.moe.fused_moe_triton.layer.is_in_tc_piecewise_cuda_graph", + return_value=True, + ): + piecewise = FusedMoE.forward(layer, self.hidden, output) + torch.testing.assert_close(piecewise.topk_ids, materialized.topk_ids) + + def test_in_range_draft_loads_experts_in_its_routing_order(self): + """Draft expert weights stay in logical order despite a permuted target map.""" + outputs = [] + for is_draft in (False, True): + method = UnquantizedFusedMoEMethod() + scope = draft_model_build_scope() if is_draft else get_parallel().override() + with ( + scope, + patch.object(method, "create_weights"), + get_parallel().override( + tp_size=1, + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + ), + ): + layer = FusedMoE( + num_experts=4, + hidden_size=2, + intermediate_size=2, + layer_id=0, + top_k=2, + quant_method=method, + ) + weights = torch.nn.Parameter(torch.zeros((4, 2, 2)), requires_grad=False) + for expert_id in range(4): + layer.weight_loader( + param=weights, + loaded_weight=torch.eye(2) * (expert_id + 1), + weight_name="experts.w2_weight", + shard_id="w2", + expert_id=expert_id, + ) + self.assertEqual( + weights[:, 0, 0].tolist(), [1, 2, 3, 4] if is_draft else [2, 4, 1, 3] + ) + router = self._router(is_draft=is_draft) + topk = router.forward_native(self.hidden, self.logits) + expert_outputs = torch.einsum( + "tkij,tj->tki", weights[topk.topk_ids], self.hidden + ) + outputs.append((expert_outputs * topk.topk_weights[..., None]).sum(dim=1)) + torch.testing.assert_close(outputs[0], outputs[1]) + expected_scale = torch.softmax(self.logits[:, :2], dim=-1) @ torch.tensor( + [1.0, 2.0] + ) + torch.testing.assert_close(outputs[1], self.hidden * expected_scale[:, None]) + + def test_identity_placement_keeps_in_kernel_routing_available(self): + identity = torch.arange(4).view(1, 4) + self.metadata.logical_to_rank_dispatch_physical_map.copy_(identity) + self.metadata.logical_to_all_physical_map[:, :, 0].copy_(identity) + self.metadata.physical_to_logical_map.copy_(identity) + with get_exec().moe.override( + enable_eplb=False, + init_expert_location="trivial", + ep_num_redundant_experts=0, + ep_dispatch_algorithm="dynamic", + ): + for output_format in ( + TopKOutputFormat.BYPASSED, + TopKOutputFormat.TRITON_KERNEL, + ): + with self.subTest(output_format=output_format): + router = self._router(output_format=output_format) + with patch.object( + topk_module, "routing", return_value=(None, None, None, None, 2) + ): + output = router.forward_cuda(self.hidden, self.logits) + self.assertEqual(output.format, output_format) + if output_format == TopKOutputFormat.BYPASSED: + self.assertIs(output.router_logits, self.logits) + for algorithm in ("fake", "lp"): + with ( + self.subTest(algorithm=algorithm), + get_exec().moe.override(ep_dispatch_algorithm=algorithm), + ): + router = self._router(output_format=TopKOutputFormat.BYPASSED) + output = router.forward_cuda(self.hidden, self.logits) + with self.assertRaisesRegex( + NotImplementedError, "placement remapping requires" + ): + _ = output.router_logits + + def test_hash_npu_keeps_valid_ids_and_zero_padded_weights(self): + router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0) + with ( + patch.object(hash_topk_module, "_is_npu", True), + envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False), + ): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(2) + ): + output = router(self.hidden, self.logits, torch.arange(4)) + self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [3, 1], [1, 2]]) + torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2))) + + def test_hash_global_shared_column_stays_outside_expert_map(self): + with get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ): + router = HashTopK( + 2, + 4, + vocab_size=4, + scoring_func="sqrtsoftplus", + routed_scaling_factor=2.0, + layer_id=0, + ) + with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False): + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=_batch(2) + ): + output = router(self.hidden, self.logits, torch.arange(4)) + self.assertEqual( + output.topk_ids.tolist(), [[2, 0, 4], [0, 3, 4], [-1, -1, -1], [-1, -1, -1]] + ) + + def test_deepseek_opaque_bypass_requires_identity_placement_and_native_policy(self): + from sglang.srt.models import deepseek_v2 + + moe = SimpleNamespace( + alt_stream=object(), + num_fused_shared_experts=0, + shared_experts=object(), + experts=SimpleNamespace(use_flashinfer_trtllm_moe=True), + _enable_a2a_moe=False, + _fuse_shared_experts_inside_sbo=False, + is_hash=False, + topk=self._router(output_format=TopKOutputFormat.BYPASSED), + ) + with ( + patch.object(deepseek_v2, "_enable_pcg_dsv2_dual_stream", True), + patch.object( + deepseek_v2, "is_in_tc_piecewise_cuda_graph", return_value=True + ), + patch.object( + deepseek_v2, + "get_moe_runner_backend", + return_value=MoeRunnerBackend.FLASHINFER_TRTLLM, + ), + get_exec().moe.override( + enable_eplb=False, + init_expert_location="trivial", + ep_num_redundant_experts=0, + ep_dispatch_algorithm=None, + ), + ): + self.assertTrue( + deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden) + ) + with get_exec().moe.override(init_expert_location="reverse"): + self.assertFalse( + deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden) + ) + moe.topk.topk_config.output_format = TopKOutputFormat.STANDARD + self.assertFalse( + deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden) + ) + + def test_lp_keeps_global_shared_columns_outside_routed_domain(self): + router = self._router(num_fused_shared_experts=1) + ids = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32) + weights = torch.tensor([[0.8, 0.2]]).repeat(4, 1) + solved = [] + + def solve(routed_ids): + self.assertEqual(routed_ids.tolist(), [[0], [1], [2], [3]]) + solved.append(routed_ids.clone()) + return torch.ones(4, 1) + + def dispatch(routed_ids, info, probability=None): + self.assertTrue(bool(((routed_ids >= 0) & (routed_ids < 4)).all())) + return info.partial_logical_to_rank_dispatch_physical_map[routed_ids] + + get_resources().lplb_solvers[0] = SimpleNamespace(solve=solve) + self.addCleanup(get_resources().lplb_solvers.pop, 0) + with ( + get_exec().moe.override(ep_dispatch_algorithm="lp"), + patch.object(topk_module, "_is_cuda", True), + patch.object(topk_module, "topk_ids_logical_to_physical", dispatch), + ): + _, info = router.routing_inputs() + output_ids, output_weights, _ = topk_module._post_process_topk_ids( + ids, + weights, + topk_config=router.topk_config, + router_logits=self.logits, + layer_id=0, + num_token_non_padded=torch.tensor(2), + expert_location_dispatch_info=info, + ) + self.assertEqual(len(solved), 1) + self.assertEqual(output_ids.tolist(), [[2, 4], [0, 4], [-1, -1], [-1, -1]]) + torch.testing.assert_close(output_weights[:2], weights[:2]) + + def test_lazy_lp_materialization_uses_its_owner_or_explicit_override(self): + """Lazy dispatcher materialization must retain the layer's solver and capture slot.""" + get_resources().expert_location_metadata = _placement(num_layers=8) + hidden = self.hidden.to(torch.bfloat16) + group = SimpleNamespace(all_gatherv=lambda tensors, sizes: tensors) + dispatcher = object.__new__(FlashinferDispatcher) + dispatcher.ep_size = 1 + dispatcher.ep_rank = 0 + with get_parallel().override(moe_ep_size=1, moe_ep_rank=0): + dispatcher.prefill_dispatcher = StandardDispatcher( + SimpleNamespace( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + num_experts=4, + num_local_experts=4, + num_fused_shared_experts=0, + ) + ) + for owner, override, expected_owner in ( + (7, None, 7), + (0, None, 0), + (7, 0, 0), + (0, 7, 7), + ): + with self.subTest(owner=owner, override=override): + solved = [] + captured = {} + + def solver(layer_id): + def solve(ids): + self.assertEqual(ids.tolist(), [[0, 1]] * 4) + solved.append(layer_id) + return torch.ones((4, 1)) + + return SimpleNamespace(solve=solve) + + class Capture: + def capture(self, *, layer_id, topk_indices): + captured[layer_id] = topk_indices.clone() + + def dispatch_probability(ids, info, probability): + return info.partial_logical_to_rank_dispatch_physical_map[ids] + + with ( + get_exec().moe.override(ep_dispatch_algorithm="lp"), + get_parallel().override( + tp_group=group, moe_ep_size=1, moe_ep_rank=0 + ), + patch.dict( + get_resources().lplb_solvers, + {0: solver(0), 7: solver(7)}, + clear=True, + ), + patch.object(get_resources(), "experts_capturer", Capture()), + patch.object(topk_module, "_is_cuda", True), + patch.object( + expert_location_dispatch, + "_topk_ids_logical_to_physical_probability", + dispatch_probability, + ), + patch( + "sglang.srt.layers.moe.token_dispatcher.flashinfer.get_dp_global_num_tokens", + return_value=None, + ), + ): + router = self._router( + layer_id=owner, output_format=TopKOutputFormat.BYPASSED + ) + output = router.forward_cuda( + hidden, self.logits, num_token_non_padded=torch.tensor(2) + ) + self.assertEqual(len(output), 5) + if override is not None: + output = output.to_standard(layer_id=override) + dispatched = dispatcher._dispatch_prefill_allgather(hidden, output) + self.assertEqual(solved, [expected_owner]) + self.assertEqual(set(captured), {expected_owner}) + self.assertEqual(captured[expected_owner].tolist(), [[0, 1]] * 4) + self.assertEqual( + dispatched.topk_output.topk_ids.tolist(), + [[2, 0], [2, 0], [-1, -1], [-1, -1]], + ) + self.assertIs(dispatched.hidden_states, hidden) + + def test_empty_target_participates_in_lp_without_caller_layer_id(self): + class Solver: + def __init__(self): + self.participations = [] + + def solve(self, ids): + self.participations.append(tuple(ids.shape)) + + solver = Solver() + get_resources().lplb_solvers[0] = solver + self.addCleanup(get_resources().lplb_solvers.pop, 0) + target = self._router() + with draft_model_build_scope(): + draft = self._router() + with get_parallel().override(tp_group=SimpleNamespace(world_size=1)): + output = target.empty_topk_output(torch.device("cpu")) + draft_output = draft.empty_topk_output(torch.device("cpu")) + self.assertEqual(output.topk_ids.shape, (0, 2)) + self.assertEqual(draft_output.topk_ids.shape, (0, 2)) + self.assertEqual(solver.participations, [(0, 2)]) + + def test_compile_reads_current_buffer(self): + router = self._router(is_draft=True) + batch = _batch(3) + + def mask_rows(x): + bound, _ = router.routing_inputs() + return x * (torch.arange(x.shape[0]) < bound) + + compiled = torch.compile(mask_rows, backend="eager", fullgraph=True) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=batch + ): + self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 1.0, 1.0, 0.0]) + batch.num_token_non_padded.fill_(1) + self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 0.0, 0.0, 0.0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py b/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py new file mode 100644 index 0000000000..a2dccd470f --- /dev/null +++ b/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py @@ -0,0 +1,342 @@ +"""GGUF expert kernels drop inactive routes during padded graph replay.""" + +import unittest +from contextlib import nullcontext +from unittest.mock import patch + +import numpy as np +import torch +from gguf import GGMLQuantizationType, dequantize, quantize + +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher +from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK +from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.layers.quantization import gguf as gguf_runtime +from sglang.srt.layers.quantization.gguf import ( + GGUFConfig, + GGUFMoEMethod, + fused_moe_gguf, + ggml_moe_get_block_size, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b", runner_config="1-gpu-small") + + +class TestGgufInactiveRoutes(CustomTestCase): + def setUp(self): + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cuda") + ) + self.enterContext( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.AUTO, a2a_backend=MoeA2ABackend.NONE + ) + ) + self.enterContext( + get_parallel().override( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + attn_dp_size=1, + attn_cp_size=1, + tp_group=None, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ) + ) + + @staticmethod + def _weights(quant_type): + rng = np.random.default_rng(491) + shape13, shape2 = (4, 512, 256), (4, 256, 256) + packed, dense = [], [] + for shape in (shape13, shape2): + weight = rng.normal(0, 0.03, shape).astype(np.float32) + qweight = quantize(weight, quant_type) + packed.append(torch.from_numpy(qweight.copy()).cuda()) + dense.append(torch.from_numpy(dequantize(qweight, quant_type)).cuda()) + return packed, dense + + @staticmethod + def _method(packed, quant_type, *, global_experts=4): + method = GGUFMoEMethod(GGUFConfig()) + layer = torch.nn.Module() + method.create_weights( + layer, + num_experts=4, + hidden_size=256, + intermediate_size_per_partition=256, + params_dtype=torch.float16, + ) + for param, weight in zip((layer.w13_qweight, layer.w2_qweight), packed): + param.materialize(weight.shape, device=weight.device, dtype=weight.dtype) + param.data.copy_(weight) + layer.w13_qweight_type.weight_type = int(quant_type) + layer.w2_qweight_type.weight_type = int(quant_type) + config = MoeRunnerConfig( + num_experts=global_experts, + num_local_experts=4, + num_fused_shared_experts=0, + hidden_size=256, + intermediate_size_per_partition=256, + top_k=2, + ) + method.create_moe_runner(layer, config) + config.runner_backend = method.get_moe_runner_backend(layer) + return method, layer, StandardDispatcher(config) + + @staticmethod + def _q8_1_matmul(inputs, weight, packed_weight, quant_type): + # GGUF GEMMs quantize each 32 input values to Q8_1, including an FP16 + # scale and input sum. CUDA roundf rounds halfway values away from zero. + blocks = inputs.float().unflatten(-1, (-1, 32)) + scale = blocks.abs().amax(dim=-1, keepdim=True) / 127 + denominator = torch.where(scale == 0, 1.0, scale) + codes = blocks.sign() * torch.floor(blocks.abs() / denominator + 0.5) + quantized = codes * scale.half().float() + result = torch.einsum( + "...bi,...nbi->...n", quantized, weight.unflatten(-1, (-1, 32)) + ) + if quant_type == GGMLQuantizationType.Q4_0: + # Q4_0 subtracts its zero point using the original FP16 input sum, + # rather than summing the dequantized Q8_1 codes. + weight_scale = ( + packed_weight.unflatten(-1, (-1, 18))[..., :2] + .contiguous() + .view(torch.float16) + .squeeze(-1) + .float() + ) + sum_error = quantized.sum(dim=-1) - blocks.sum(dim=-1).half().float() + result += 8 * torch.einsum("...b,...nb->...n", sum_error, weight_scale) + return result.to(inputs.dtype) + + @classmethod + def _reference(cls, hidden, logits, dense, packed, quant_type): + weights, ids = logits.float().softmax(dim=-1).topk(2, dim=-1) + projected = cls._q8_1_matmul( + hidden.unsqueeze(1), dense[0][ids], packed[0][ids], quant_type + ) + gate, up = projected.float().chunk(2, dim=-1) + activated = (torch.nn.functional.silu(gate) * up).to(hidden.dtype) + down = cls._q8_1_matmul(activated, dense[1][ids], packed[1][ids], quant_type) + weighted = (down.float() * weights.unsqueeze(-1)).to(hidden.dtype) + return weighted.sum(dim=1).float() + + def test_block_size_metadata_without_tensor_arguments(self): + # Runtime metadata must work without a Tensor-dispatched wheel operator. + for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0): + with self.subTest(quant_type=quant_type): + self.assertIn(ggml_moe_get_block_size(int(quant_type)), (4, 8)) + + @torch.inference_mode() + def test_padded_graph_replay_matches_unpadded_experts(self): + """Changing the live count must zero dropped GGUF routes on every replay.""" + for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0): + packed, dense = self._weights(quant_type) + for tokens, partial in ((64, 48), (96, 71)): + with self.subTest(quant_type=quant_type, tokens=tokens): + self._check_graph(packed, dense, quant_type, tokens, partial) + + def _check_graph(self, packed, dense, quant_type, tokens, partial): + method, layer, dispatcher = self._method(packed, quant_type) + topk = TopK( + top_k=2, layer_id=0, renormalize=False, allow_routed_experts_capture=False + ) + topk.bind_runner_backend(method.get_moe_runner_backend(layer)) + generator = torch.Generator(device="cuda").manual_seed(tokens) + base_hidden = torch.randn( + tokens, 256, generator=generator, device="cuda" + ).half() + base_logits = torch.randn(tokens, 4, generator=generator, device="cuda") + logits = base_logits.clone() + hidden = base_hidden.clone() + live_count = torch.tensor(tokens, device="cuda", dtype=torch.int32) + batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=1, + input_ids=torch.zeros(tokens, device="cuda", dtype=torch.int64), + req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64), + seq_lens=torch.tensor([tokens], device="cuda", dtype=torch.int32), + out_cache_loc=torch.arange(tokens, device="cuda"), + seq_lens_sum=tokens, + positions=torch.arange(tokens, device="cuda"), + num_token_non_padded=live_count, + global_num_token_non_padded=live_count, + ) + + def run(): + with forward_context( + ForwardContext(attn_backend=None), forward_batch=batch + ): + routes = topk.forward_cuda(hidden, logits) + dispatched = dispatcher.dispatch(hidden, routes) + combined = method.apply(layer, dispatched) + return dispatcher.combine(combined), routes.topk_ids + + native_outputs = [] + native_mmq = gguf_runtime.ggml_moe_a8 + + native_align = gguf_runtime.moe_align_block_size + + def poison_alignment(*args, **kwargs): + sorted_ids, expert_ids, count = native_align(*args, **kwargs) + block_starts = ( + torch.arange(expert_ids.numel(), device=expert_ids.device) * 4 + ) + expert_ids.masked_fill_(block_starts >= count, 4) + return sorted_ids, expert_ids, count + + def record_mmq_outputs(*args, **kwargs): + if not torch.cuda.is_current_stream_capturing(): + # Fail before native dispatch if a slack block can index expert E. + self.assertTrue(bool((args[3] < args[1].shape[0]).all())) + result = native_mmq(*args, **kwargs) + if torch.cuda.is_current_stream_capturing(): + native_outputs.append(result) + return result + + # Observe the real provider's returned buffers without replacing its + # computation. Poisoning them makes a skipped MMQ store deterministic. + observe_mmq = ( + patch.object(gguf_runtime, "ggml_moe_a8", record_mmq_outputs) + if tokens > 64 + else nullcontext() + ) + with ( + observe_mmq, + patch.object(gguf_runtime, "moe_align_block_size", poison_alignment), + ): + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + run() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output, ids = run() + self.assertEqual(len(native_outputs), 2 if tokens > 64 else 0) + # Dropped slots must overwrite poisoned native intermediates rather + # than inherit any previous live route or allocator contents. + for live in (tokens, partial, 0, partial // 2, tokens): + hidden.copy_(base_hidden) + hidden[live:].fill_(float("nan")) + logits.copy_(base_logits) + logits[live:].fill_(float("nan")) + live_count.fill_(live) + output.fill_(float("nan")) + for intermediate in native_outputs: + intermediate.fill_(float("nan")) + graph.replay() + for intermediate in native_outputs: + self.assertTrue(torch.isfinite(intermediate).all().item()) + inactive = intermediate.view(tokens, 2, -1)[live:] + self.assertEqual(torch.count_nonzero(inactive).item(), 0) + self.assertTrue(torch.isfinite(output).all().item()) + self.assertTrue((ids[live:] == -1).all().item()) + self.assertEqual(torch.count_nonzero(output[live:]).item(), 0) + if live: + expected = self._reference( + base_hidden[:live], base_logits[:live], dense, packed, quant_type + ) + self.assertGreater(expected.abs().max().item(), 1e-3) + torch.testing.assert_close( + output[:live].float(), expected, atol=2e-3, rtol=0.06 + ) + + @torch.inference_mode() + def test_nonlocal_routes_preserve_local_contributions(self): + """A nonlocal slot must not erase the same token's valid expert result.""" + packed, dense = self._weights(GGMLQuantizationType.Q4_0) + with get_parallel().override( + tp_size=2, + tp_rank=1, + attn_tp_size=2, + attn_tp_rank=1, + moe_ep_size=2, + moe_ep_rank=1, + ): + method, layer, dispatcher = self._method( + packed, GGMLQuantizationType.Q4_0, global_experts=8 + ) + hidden = torch.full((3, 256), 0.25, device="cuda", dtype=torch.float16) + ids = torch.tensor([[4, 1], [-1, 5], [0, 2]], device="cuda", dtype=torch.int32) + weights = torch.tensor( + [[0.7, float("nan")], [float("nan"), 0.6], [float("nan"), float("nan")]], + device="cuda", + ) + hidden[2].fill_(float("nan")) + routes = StandardTopKOutput(weights, ids, None) + output = dispatcher.combine( + method.apply(layer, dispatcher.dispatch(hidden, routes)) + ) + self.assertTrue(torch.isfinite(output).all().item()) + self.assertEqual(torch.count_nonzero(output[2]).item(), 0) + for token, expert, scale in ((0, 0, 0.7), (1, 1, 0.6)): + projected = self._q8_1_matmul( + hidden[token], + dense[0][expert], + packed[0][expert], + GGMLQuantizationType.Q4_0, + ) + gate, up = projected.float().chunk(2) + activated = (torch.nn.functional.silu(gate) * up).half() + expected = ( + ( + self._q8_1_matmul( + activated, + dense[1][expert], + packed[1][expert], + GGMLQuantizationType.Q4_0, + ).float() + * scale + ) + .half() + .float() + ) + torch.testing.assert_close( + output[token].float(), expected, atol=2e-3, rtol=0.06 + ) + + @torch.inference_mode() + def test_unquantized_fallback_drops_negative_ids(self): + """The slow path must not alias a dropped route to the last expert.""" + generator = torch.Generator(device="cuda").manual_seed(17) + w13 = torch.randn(2, 32, 16, generator=generator, device="cuda").half() / 16 + w2 = torch.randn(2, 16, 16, generator=generator, device="cuda").half() / 16 + hidden = torch.ones(2, 16, device="cuda", dtype=torch.float16) + hidden[1].fill_(float("nan")) + ids = torch.tensor([[0, -1], [-1, -1]], device="cuda", dtype=torch.int32) + weights = torch.tensor( + [[1.0, float("nan")], [float("nan"), float("nan")]], device="cuda" + ) + output = fused_moe_gguf( + hidden, + w13, + w2, + weights, + ids, + GGMLQuantizationType.F16, + GGMLQuantizationType.F16, + "silu", + ) + projected = hidden[0] @ w13[0].t() + gate, up = projected.chunk(2) + expected = (torch.nn.functional.silu(gate) * up) @ w2[0].t() + self.assertTrue(torch.isfinite(output).all().item()) + self.assertEqual(torch.count_nonzero(output[1]).item(), 0) + torch.testing.assert_close(output[0], expected, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/quantization/test_mxfp4_sm100_trtllm_gen.py b/test/registered/unit/layers/quantization/test_mxfp4_sm100_trtllm_gen.py index 60e8f4e361..37788684fd 100644 --- a/test/registered/unit/layers/quantization/test_mxfp4_sm100_trtllm_gen.py +++ b/test/registered/unit/layers/quantization/test_mxfp4_sm100_trtllm_gen.py @@ -196,7 +196,7 @@ class _MockDispatchOutput: self.hidden_states = hidden_states self.topk_output = BypassedTopKOutput( hidden_states=hidden_states, - router_logits=router_logits, + raw_router_logits=router_logits, topk_config=TopKConfig(top_k=top_k, renormalize=True), ) diff --git a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py index d92c7df229..85d2d16417 100644 --- a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py +++ b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py @@ -189,6 +189,7 @@ def _run_marlin_flow( fused_tails=0, marlin_is_ep=[], marlin_output_ids=[], + marlin_output_initialized=[], align_num_experts=[], schedule=[], zeroed=[], @@ -248,6 +249,15 @@ def _run_marlin_flow( return _ORIGINAL_TENSOR_ZERO(tensor, *args, **kwargs) monkeypatch.setattr(torch.Tensor, "zero_", tracked_zero) + original_empty = torch.empty + + def poisoned_empty(*args, **kwargs): + result = original_empty(*args, **kwargs) + if result.is_floating_point(): + result.fill_(float("nan")) + return result + + monkeypatch.setattr(torch, "empty", poisoned_empty) def merged_experts_fused_moe_lora_add(**kwargs): intermediate = kwargs.get("intermediate_buffer") @@ -298,11 +308,13 @@ def _run_marlin_flow( def fake_marlin_gemm(_x, output, *_args, **kwargs): calls.marlin_is_ep.append(kwargs["is_ep"]) calls.marlin_output_ids.append(id(output)) + calls.marlin_output_initialized.append( + bool(torch.isfinite(output).all() and torch.count_nonzero(output) == 0) + ) calls.schedule.append( ("marlin", stream_state["current"].name, len(calls.marlin_is_ep)) ) - # ``fill_``, never ``zero_``: the zero-fill contracts are asserted by - # watching ``Tensor.zero_`` calls, so the fakes must not forge one. + # Inspect initialization before the fake GEMM writes its result. output.fill_(0) return output @@ -410,22 +422,14 @@ def _run_marlin_flow( def test_ep_uses_local_alignment_and_skips_nonlocal_marlin_blocks(monkeypatch): - """Guards expert-parallel index localization and both accumulator clears; - reds when ``moe_align_block_size`` gets the global expert count, ``is_ep`` - is dropped on a Marlin GEMM, or an accumulator is left uninitialized.""" + """EP alignment stays local and the down accumulator starts finite and zero.""" calls = _run_marlin_flow(monkeypatch, tokens=16, ep=True, slots=2, rank=128) assert calls.align_num_experts == [1] assert calls.marlin_is_ep == [True, True] - down_gemm_output = calls.marlin_output_ids[1] - assert [item[2] for item in calls.zeroed] == [down_gemm_output] - zero_index = calls.schedule.index(("zero", "main", down_gemm_output)) - down_gemm_index = [ - index for index, item in enumerate(calls.schedule) if item[0] == "marlin" - ][1] - assert zero_index < down_gemm_index + assert calls.marlin_output_initialized[1] down_stage = [ call for call in calls.merged if call["stage"] == "all" and call["shared_b"] diff --git a/test/registered/unit/model_executor/runner/test_graph_routing_context.py b/test/registered/unit/model_executor/runner/test_graph_routing_context.py new file mode 100644 index 0000000000..085260f2af --- /dev/null +++ b/test/registered/unit/model_executor/runner/test_graph_routing_context.py @@ -0,0 +1,350 @@ +"""Executor-owned routing bounds survive retained callbacks and padded CPU replay.""" + +import importlib +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.environ import envs +from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds +from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.moe import topk as topk_module +from sglang.srt.layers.moe.topk import TopK, fused_topk_torch_native +from sglang.srt.layers.moe.utils import initialize_moe_config +from sglang.srt.model_executor.cpu_graph_runner import CPUGraphRunner +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, + PPProxyTensors, + enable_num_token_non_padded, +) +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + get_forward_context, + has_forward_context, +) +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( + DecodeCudaGraphRunner, +) +from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import ( + BreakableCudaGraphBackend, +) +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +breakable_graph = importlib.import_module( + "sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph" +) +breakable_backend = importlib.import_module( + "sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend" +) + + +def _batch(size, live, *, prepared=True): + batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=size, + input_ids=torch.arange(size), + req_pool_indices=torch.arange(size), + seq_lens=torch.ones(size, dtype=torch.int32), + out_cache_loc=torch.arange(size), + seq_lens_sum=size, + positions=torch.arange(size), + capture_hidden_mode=CaptureHiddenMode.NULL, + ) + batch.global_num_token_non_padded = torch.tensor(live, dtype=torch.int32) + batch.global_num_token_non_padded_cpu = live + batch.num_token_non_padded = batch.global_num_token_non_padded if prepared else None + batch.global_dp_buffer_len = size + batch.dp_padding_mode = DpPaddingMode.SUM_LEN + return batch + + +def _attention_backend(): + return SimpleNamespace( + init_forward_metadata=lambda batch: None, + init_forward_metadata_out_graph=lambda batch, **kwargs: None, + init_forward_metadata_in_graph=lambda batch: None, + shared_read_ends=lambda mode: SharedReadEnds.UNKNOWN, + ) + + +class _RoutingModel: + shared_experts_config = SharedExpertsConfig() + + def __init__(self, *, proxy_output): + self.topk = TopK( + 2, + layer_id=0, + is_draft=True, + custom_routing_function=fused_topk_torch_native, + ) + self.proxy_output = proxy_output + self.seen = [] + self.fail = False + + def forward(self, input_ids, positions, batch): + rows = len(input_ids) + bound, _ = self.topk.routing_inputs() + ids = self.topk.forward_native( + torch.ones((rows, 2)), + torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(rows, 1), + ).topk_ids + self.seen.append( + (batch, bound, get_forward_context().attn_backend, ids.clone()) + ) + if self.fail: + raise RuntimeError("replayed model failed") + if self.proxy_output: + return PPProxyTensors({"routing_ids": ids}) + return LogitsProcessorOutput(next_token_logits=ids.float()) + + +class _CpuStream: + def wait_stream(self, other): + pass + + +class _NoDeviceGraph: + """Only accelerator graph recording/replay is replaced; eager breaks remain real.""" + + def capture_begin(self, **kwargs): + pass + + def capture_end(self): + pass + + def replay(self): + pass + + +class TestGraphRoutingContext(CustomTestCase): + def setUp(self): + super().setUp() + config = get_context().override_server_args( + enable_eplb=False, ep_dispatch_algorithm=None, debug_cuda_graph=True + ) + config.install() + self.addCleanup(config.restore) + initialize_moe_config() + topology = get_parallel().override( + tp_size=4, + moe_ep_size=4, + attn_dp_size=1, + attn_cp_size=1, + attn_tp_size=4, + attn_tp_rank=0, + tp_group=SimpleNamespace(barrier=lambda: None, world_size=4), + ) + topology.__enter__() + self.addCleanup(topology.__exit__, None, None, None) + for name in ("_is_cuda", "_is_hip", "_use_aiter"): + scope = patch.object(topk_module, name, False) + scope.start() + self.addCleanup(scope.stop) + + def _model_runner(self, *, device, model): + runner = ModelRunner.__new__(ModelRunner) + runner.device = device + runner.attn_backend = _attention_backend() + runner.hisparse_coordinator = None + runner.device_timer = None + runner.is_draft_worker = False + runner.spec_algorithm = SpeculativeAlgorithm.NONE + runner.capture_tail_hooks = [] + runner.canary_manager = None + runner.model = model + return runner + + def _debug_runner(self): + model = _RoutingModel(proxy_output=True) + model_runner = self._model_runner(device="cuda", model=model) + runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner) + runner.model_runner = model_runner + runner.captured_req_width = 1 + runner.ragged_verify_mode = False + runner.enable_pdmux = False + runner.pp_size = 1 + runner.dllm_uses_input_embeds = False + runner.in_graph_metadata_prep_done = None + runner.is_dllm = False + static = _batch(4, 4) + runner.attn_backend = captured_backend = _attention_backend() + runner.capture_prepare = lambda *args, **kwargs: ( + static, + captured_backend, + None, + ) + runner.tbo_plugin = SimpleNamespace( + capture_one_batch_size=lambda *args, **kw: None + ) + runner.deepep_adapter = SimpleNamespace(capture=lambda **kwargs: None) + stream = _CpuStream() + runner.device_module = SimpleNamespace( + Stream=_CpuStream, + current_stream=lambda *args: stream, + synchronize=lambda: None, + is_current_stream_capturing=lambda: False, + ) + runner.backend = BreakableCudaGraphBackend(runner, debug_eager=True) + runner.can_run_graph = lambda batch: True + + def load_batch(batch, pp_proxy_tensors): + # Substitute only the accelerator input copies/shape selection. + static.num_token_non_padded.copy_(batch.global_num_token_non_padded) + runner.raw_bs = runner.raw_num_token = batch.batch_size + runner.bs = 4 + runner._replay_graph_key = runner._make_graph_key(4) + + runner.load_batch = load_batch + model_runner.decode_cuda_graph_runner = runner + for target, name, replacement in ( + (torch.cuda, "CUDAGraph", _NoDeviceGraph), + (breakable_graph, "get_device_module", lambda: runner.device_module), + (breakable_backend, "graph_pool_capture_scope", nullcontext), + (breakable_backend, "graph_pool_replay_scope", nullcontext), + ): + scope = patch.object(target, name, replacement) + scope.start() + self.addCleanup(scope.stop) + with envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE.override(False): + runner.capture_one_shape(4, model.forward) + self.assertEqual( + len(runner.backend._graphs[runner._make_graph_key(4)]._break_fns), 1 + ) + return model_runner, model, static, captured_backend + + def _cpu_runner(self): + model = _RoutingModel(proxy_output=False) + model_runner = self._model_runner(device="cpu", model=model) + runner = CPUGraphRunner.__new__(CPUGraphRunner) + runner.model_runner = model_runner + runner.is_encoder_decoder = False + runner.capture_hidden_mode = CaptureHiddenMode.NULL + runner.capture_bs = [4] + runner.captured_req_width = 1 + runner.seq_len_fill_value = 1 + runner.graphs = {4: model.forward} + runner.graphs_cross = {} + static = _batch(4, 4) + runner.captured_forward_batches = {4: static} + runner.can_run_graph = lambda batch: True + model_runner.decode_cuda_graph_runner = runner + return model_runner, model, static, model_runner.attn_backend + + def _capture_cpu_runner(self): + model_runner, model, _, backend = self._cpu_runner() + runner = model_runner.decode_cuda_graph_runner + runner.capture_bs = [1, 4] + runner.capture_forward_mode = ForwardMode.DECODE + runner.captured_forward_batches = {} + runner.captured_forward_batches_cross = {} + runner.input_ids = torch.arange(4) + runner.req_pool_indices = torch.arange(4) + runner.seq_lens = torch.ones(4, dtype=torch.int32) + runner.out_cache_loc = torch.arange(4) + runner.positions = torch.arange(4) + runner.mrope_positions = torch.zeros((3, 4), dtype=torch.int64) + runner.num_token_non_padded = torch.zeros(1, dtype=torch.int64) + model_runner.kv_index_translator = SimpleNamespace( + bind_runner_slots=lambda batch: None + ) + backend.init_forward_metadata_capture_cpu_graph = lambda *args: None + for size in reversed(runner.capture_bs): + runner.graphs[size], _ = runner.capture_one_batch_size( + size, model.forward, skip_cross_attention=True + ) + return runner, model, backend + + def test_cpu_capture_without_tracking_keeps_live_routes(self): + """Padded CPU replays keep every live route when count tracking is disabled.""" + with get_parallel().override( + tp_size=1, + moe_ep_size=1, + attn_tp_size=1, + tp_group=SimpleNamespace(barrier=lambda: None, world_size=1), + ): + self.assertFalse(enable_num_token_non_padded()) + runner, model, backend = self._capture_cpu_runner() + for size in (3, 4, 2, 1): + with self.subTest(size=size): + incoming = _batch(size, size, prepared=False) + incoming.global_num_token_non_padded = None + with forward_context( + ForwardContext(attn_backend=backend), forward_batch=incoming + ): + eager = model.forward( + incoming.input_ids, incoming.positions, incoming + ) + output = runner.execute(incoming) + torch.testing.assert_close( + output.next_token_logits, eager.next_token_logits + ) + prepared, bound, _, ids = model.seen[-1] + self.assertIsNone(bound) + self.assertTrue((ids >= 0).all()) + self.assertEqual(prepared.batch_size, 4 if size > 1 else 1) + self.assertFalse(has_forward_context()) + for captured in runner.captured_forward_batches.values(): + self.assertIsNone(captured.num_token_non_padded) + + def _assert_replay(self, model_runner, model, static, backend): + persistent_bound = static.num_token_non_padded + for live in (3, 1, 0): + incoming = _batch(live, live, prepared=False) + model_runner._forward_raw(incoming, None) + batch, bound, active_backend, ids = model.seen[-1] + self.assertIs(batch, static) + self.assertIs(bound, persistent_bound) + self.assertEqual(int(bound), live) + self.assertIs(active_backend, backend) + self.assertEqual(ids[:live].tolist(), [[0, 1]] * live) + self.assertEqual(ids[live:].tolist(), [[-1, -1]] * (4 - live)) + self.assertIsNone(incoming.num_token_non_padded) + self.assertFalse(has_forward_context()) + + def test_debug_saved_forward_rebinds_static_batch(self): + self._assert_replay(*self._debug_runner()) + + def test_cpu_replay_rebinds_actual_prepared_batch(self): + self._assert_replay(*self._cpu_runner()) + + def _assert_failure_restores_parent(self, model_runner, model, static, backend): + parent = _batch(1, 1) + with forward_context( + ForwardContext(attn_backend=object()), forward_batch=parent + ): + previous = get_forward_context() + model.fail = True + with self.assertRaisesRegex(RuntimeError, "replayed model failed"): + model_runner._forward_raw(_batch(3, 3, prepared=False), None) + self.assertIs(model.seen[-1][0], static) + self.assertIs(model.seen[-1][1], static.num_token_non_padded) + self.assertIs(model.seen[-1][2], backend) + self.assertIs(get_forward_context(), previous) + self.assertIs( + previous.moe_num_token_non_padded(), parent.num_token_non_padded + ) + self.assertFalse(has_forward_context()) + + def test_debug_saved_forward_failure_restores_parent(self): + self._assert_failure_restores_parent(*self._debug_runner()) + + def test_cpu_replay_failure_restores_parent(self): + self._assert_failure_restores_parent(*self._cpu_runner()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py index ba21567196..aec6aa3d23 100644 --- a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py +++ b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py @@ -22,12 +22,13 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch import torch +import sglang.srt.models.deepseek_v4_replay_tail as replay_tail from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.model_executor.forward_batch_info import ( @@ -37,6 +38,15 @@ from sglang.srt.model_executor.forward_batch_info import ( compute_local_num_token_non_padded_cpu, enable_num_token_non_padded, ) +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + get_forward_context, + has_forward_context, +) +from sglang.srt.model_executor.runner.row_bucket_graph_runner import ( + RowBucketGraphRunner, +) from sglang.srt.runtime_context import get_flags, get_parallel from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -254,5 +264,185 @@ class TestEnableNumTokenNonPadded(CustomTestCase): self.assertIs(enable_num_token_non_padded(), expected) +class TestRowBucketMoeBound(CustomTestCase): + def test_capture_owns_the_moe_bound_and_restores_the_outer_context(self): + """A bucket's recorded routing must retain its own mutable count buffer.""" + for active_context in (False, True): + with ( + self.subTest(active_context=active_context), + get_parallel().override(attn_dp_size=1, attn_cp_size=1), + sparse_moe_input("local"), + ): + outer_batch = _forward_batch(sharded=False, local=3, glob=3) + outer = ForwardContext(attn_backend=None) + scope = ( + forward_context(outer, forward_batch=outer_batch) + if active_context + else nullcontext() + ) + observed = [] + + def body(inputs, capture_batch): + if active_context: + bound = get_forward_context().moe_num_token_non_padded() + self.assertIs(bound, capture_batch.num_token_non_padded) + observed.append(bound) + else: + self.assertFalse(has_forward_context()) + return inputs + + def record(body, rows, count, batch, start, free_before): + body() + return SimpleNamespace(batch=(), num_token_non_padded=count) + + runner = RowBucketGraphRunner(row_step=8, max_rows=8, label="test") + with ( + scope, + patch.object(torch.cuda, "mem_get_info", return_value=(1, 2)), + patch.object(runner, "_record", side_effect=record), + ): + graph = runner._capture( + 8, + 3, + [torch.zeros(8, 1)], + outer_batch, + body, + lambda batch, inputs: nullcontext(), + ) + graph.num_token_non_padded.fill_(7) + outer_batch.num_token_non_padded.fill_(1) + if active_context: + self.assertEqual(int(observed[0]), 7) + self.assertIs( + get_forward_context().moe_num_token_non_padded(), + outer_batch.num_token_non_padded, + ) + self.assertFalse(has_forward_context()) + + +class TestDecoderReplayTailMoeBound(CustomTestCase): + # A row graph replaces LOCAL only, so even equal-valued or aliased GLOBAL + # buffers cannot establish that the captured getter reads its own count. + _CASES = ( + ("local", False, 3, 7, False, True), + ("global", True, 3, 7, False, False), + ("equal_global", True, 3, 3, False, False), + ("aliased_local", False, 3, 3, True, False), + ("aliased_global", True, 3, 3, True, False), + ("unbounded_global", True, 3, None, False, True), + ("absent_local", True, None, 7, False, True), + ) + + @staticmethod + def _batch_and_tail(sharded, local, glob, alias): + batch = _forward_batch( + sharded=sharded, local=local, glob=glob, forward_mode=ForwardMode.EXTEND + ) + batch.input_ids = torch.arange(4) + if alias: + batch.global_num_token_non_padded = batch.num_token_non_padded + tail = SimpleNamespace( + positions=torch.arange(3), cp_metadata=None, rows=lambda t: t[:3] + ) + return batch, tail + + def test_eager_prefill_declines_a_separate_moe_count_buffer(self): + for label, sharded, local, glob, alias, expected in self._CASES: + with ( + self.subTest(bound=label), + get_parallel().override(attn_dp_size=1, attn_cp_size=1), + sparse_moe_input("ffn"), + ): + batch, tail = self._batch_and_tail(sharded, local, glob, alias) + original_global = batch.global_num_token_non_padded + tail_layers = replay_tail.DecoderReplayTail.__new__( + replay_tail.DecoderReplayTail + ) + tail_layers.runner = SimpleNamespace( + ready=Mock(), warm=True, bucket_rows=lambda n: 8 + ) + with ( + patch.object( + replay_tail, "get_is_capture_mode", return_value=False + ), + patch.object( + replay_tail, + "get_global_expert_distribution_recorder", + return_value=SimpleNamespace(recording=False), + ), + patch( + "sglang.srt.layers.attention.dsv4.unified_kv_gate.is_unified_kv_triton", + return_value=False, + ), + ): + use_graph, dp_rows = tail_layers.plan( + tail, + forward_batch=batch, + num_tokens=4, + tail_in_graph=False, + late_dp_counts=None, + ) + self.assertEqual(use_graph, expected) + self.assertIsNone(dp_rows) + self.assertIs(batch.global_num_token_non_padded, original_global) + + def test_prefill_graph_break_keeps_a_separate_moe_count_eager(self): + for label, sharded, local, glob, alias, expected in self._CASES: + with ( + self.subTest(bound=label), + get_parallel().override(attn_dp_size=1, attn_cp_size=1), + sparse_moe_input("ffn"), + ): + batch, tail = self._batch_and_tail(sharded, local, glob, alias) + original_global = batch.global_num_token_non_padded + output = torch.tensor([11]) + tail_layers = SimpleNamespace( + runner=SimpleNamespace(warm=True, bucket_rows=lambda n: 8), + _can_capture_moe_bound=( + replay_tail.DecoderReplayTail._can_capture_moe_bound + ), + run=Mock(return_value=(output, None, [])), + _run_layers_eagerly=Mock(return_value=(output, None, [])), + _bridge_outputs=lambda rows, outputs: tuple(outputs), + ) + backend = SimpleNamespace( + forward_metadata=SimpleNamespace(late_layer_tail=tail), + exit_late_layer_tail=Mock(), + ) + with ( + patch.object(replay_tail, "get_attn_backend", return_value=backend), + patch.object( + replay_tail, + "get_tc_piecewise_forward_context", + return_value=SimpleNamespace(forward_batch=batch), + ), + patch.object( + replay_tail, + "_enter_eager_tail_in_prefill_graph", + return_value=(None, batch), + ), + get_flags().dp.override(capturing_prefill_graph=False), + ): + outputs = replay_tail._tail_layers_in_prefill_graph( + tail_layers, + lambda tensors: tensors, + False, + 4, + None, + False, + [torch.zeros(4, 1)], + torch.arange(4), + batch.input_ids, + batch.input_ids, + ) + self.assertIs(outputs[0], output) + self.assertEqual(tail_layers.run.call_count, expected) + self.assertEqual( + tail_layers._run_layers_eagerly.call_count, not expected + ) + backend.exit_late_layer_tail.assert_called_once_with(None, batch) + self.assertIs(batch.global_num_token_non_padded, original_global) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_draft_entry_hook_parity.py b/test/registered/unit/models/test_draft_entry_hook_parity.py index 36c42e03ef..a4c0db4a26 100644 --- a/test/registered/unit/models/test_draft_entry_hook_parity.py +++ b/test/registered/unit/models/test_draft_entry_hook_parity.py @@ -44,6 +44,11 @@ DRAFT_SUFFIX = re.compile(r"(NextN|MTP|DSpark|DFlash|Standalone)$") def _target_of(arch: str, archs: dict): """The arch a draft entry class drafts for, if it is named after it.""" + # Bailing shares its NextN entry class across generations. Its loader + # fusion hooks belong to V3; legacy checkpoints return a fused count of 0 + # (covered by TestBailingMoeV3Gate in test_shared_experts_fusion_gates.py). + if arch == "BailingMoeForCausalLMNextN": + return "BailingMoeV3ForCausalLM" match = DRAFT_SUFFIX.search(arch) if not match: return None diff --git a/test/registered/unit/models/test_longcat_flash_router_hpc_gemm.py b/test/registered/unit/models/test_longcat_flash_router_hpc_gemm.py index 754263be1e..7d2323ee10 100644 --- a/test/registered/unit/models/test_longcat_flash_router_hpc_gemm.py +++ b/test/registered/unit/models/test_longcat_flash_router_hpc_gemm.py @@ -11,7 +11,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() -from sglang.srt.models.longcat_flash import LongcatFlashRouter # noqa: E402 +from sglang.srt.models.longcat_flash import ( # noqa: E402 + LongcatFlashMoE, + LongcatFlashRouter, +) +from sglang.srt.runtime_context import get_context, get_exec, get_parallel # noqa: E402 register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -25,6 +29,37 @@ def _longcat_config(hidden_size, n_routed_experts, *, router_bias=False): class TestLongcatFlashRouterHpcGemm(CustomTestCase): + def test_zero_experts_reject_placement_before_loading_expert_weights(self): + """Identity expert IDs cannot index routed-only placement tables.""" + config = SimpleNamespace( + routed_scaling_factor=1.0, + n_routed_experts=4, + moe_topk=2, + zero_expert_num=2, + zero_expert_type="identity", + rounter_params_dtype="float32", + hidden_act="silu", + ) + with get_context().override_server_args(model_path="dummy", device="cpu"): + for algorithm in ("static", "dynamic", "fake", "lp"): + with ( + self.subTest(algorithm=algorithm), + get_exec().moe.override(ep_dispatch_algorithm=algorithm), + self.assertRaisesRegex(ValueError, "LongCat zero experts"), + ): + LongcatFlashMoE(config=config, layer_id=0) + + with ( + get_exec().moe.override(ep_dispatch_algorithm=None), + get_parallel().override(tp_size=1), + patch( + "sglang.srt.models.longcat_flash.LongcatFlashRouter", + side_effect=RuntimeError("placement admitted"), + ), + self.assertRaisesRegex(RuntimeError, "placement admitted"), + ): + LongcatFlashMoE(config=config, layer_id=0) + def _assert_dispatches_to_hpc_gemm( self, *,