From aa0ce20201f7fa3edefb8e5f6736151a42f1a3e9 Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" <280536569+jiejingzhangamd@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:19:37 -0700 Subject: [PATCH] [ROCm] GLM-5.2 decode path: decode-shaped MoE/MLA tiles, split speculative softmax, and bf16 GEMM routing (#41725) Co-authored-by: kyle-256 --- .../ops/moe/fused_moe_triton_kernels.py | 27 ++- .../sglang/kernels/ops/sampling/__init__.py | 15 +- .../ops/sampling/temperature_softmax.py | 170 ++++++++++++++++++ python/sglang/srt/layers/moe/topk.py | 42 ++--- .../sglang/srt/layers/quantization/unquant.py | 35 ++++ .../forward_mla_rocm.py | 46 ++++- python/sglang/srt/speculative/eagle_utils.py | 9 +- python/sglang/srt/speculative/spec_utils.py | 10 +- .../ops/sampling/test_temperature_softmax.py | 75 ++++++++ .../test_fused_append_shared_experts_top6.py | 25 +++ .../registered/moe/test_topk_padded_region.py | 27 +++ .../quantization/test_aiter_a16w16_gate.py | 46 +++++ .../test_rocm_mla_absorb_fusion_gate.py | 35 ++++ 13 files changed, 524 insertions(+), 38 deletions(-) create mode 100644 python/sglang/kernels/ops/sampling/temperature_softmax.py create mode 100644 test/registered/kernels/ops/sampling/test_temperature_softmax.py create mode 100644 test/registered/unit/layers/quantization/test_aiter_a16w16_gate.py diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index 73f722a16d..592dcf3197 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -1288,10 +1288,12 @@ def _fused_append_shared_experts_kernel( out_weights_ptr, N_BASE, # runtime scalar scale_factor, # runtime scalar + num_token_non_padded_ptr, # 1-elem int tensor; only read when HAS_PADDING K: tl.constexpr, S: tl.constexpr, BLOCK_K: tl.constexpr, BLOCK_S: tl.constexpr, + HAS_PADDING: tl.constexpr, ): """ for m in range(M): @@ -1317,22 +1319,35 @@ def _fused_append_shared_experts_kernel( ids = tl.load(topk_ids_ptr + ids_row_ptr + offs_k, mask=mask_k) ws = tl.load(topk_weights_ptr + w_row_ptr + offs_k, mask=mask_k) - tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k) - tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k) - offs_s = tl.arange(0, BLOCK_S) mask_s = offs_s < S shared_ids = tl.cast(N_BASE + offs_s, ids.dtype) shared_ws = tl.full([BLOCK_S], scale_factor, dtype=ws.dtype) + if HAS_PADDING: + # Padded rows zero routed ids and all weights. + if pid >= tl.load(num_token_non_padded_ptr): + ids = tl.zeros([BLOCK_K], dtype=ids.dtype) + ws = tl.zeros([BLOCK_K], dtype=ws.dtype) + shared_ws = tl.zeros([BLOCK_S], dtype=ws.dtype) + + tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k) + tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k) + tl.store(out_ids_ptr + out_ids_row_ptr + K + offs_s, shared_ids, mask=mask_s) tl.store(out_weights_ptr + out_w_row_ptr + K + offs_s, shared_ws, mask=mask_s) def fused_append_shared_experts( - topk_ids, topk_weights, num_fused_shared_experts, scale_factor, N=None + topk_ids, + topk_weights, + num_fused_shared_experts, + scale_factor, + N=None, + num_token_non_padded=None, ): + """Append shared experts and optionally materialize padded rows.""" assert N is not None, "N (shared expert base id) must be provided" m, k = topk_ids.shape s = int(num_fused_shared_experts) @@ -1344,6 +1359,8 @@ def fused_append_shared_experts( (m, k + s), dtype=topk_weights.dtype, device=topk_weights.device ) + has_padding = num_token_non_padded is not None + ntnp_ptr = num_token_non_padded if has_padding else topk_ids _fused_append_shared_experts_kernel[(m,)]( topk_ids, topk_weights, @@ -1351,10 +1368,12 @@ def fused_append_shared_experts( out_weights, N_BASE=N, scale_factor=scale_factor, + num_token_non_padded_ptr=ntnp_ptr, K=k, S=s, BLOCK_K=triton.next_power_of_2(k), BLOCK_S=triton.next_power_of_2(s), + HAS_PADDING=has_padding, num_warps=1, ) return out_ids, out_weights diff --git a/python/sglang/kernels/ops/sampling/__init__.py b/python/sglang/kernels/ops/sampling/__init__.py index 81bd97becf..c585956d27 100644 --- a/python/sglang/kernels/ops/sampling/__init__.py +++ b/python/sglang/kernels/ops/sampling/__init__.py @@ -40,10 +40,21 @@ def cake_softmax(logits: torch.Tensor) -> torch.Tensor: return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits) -def softmax(logits: torch.Tensor) -> torch.Tensor: - """Softmax for sampler logits, with a qualified Blackwell fast path.""" +def softmax( + logits: torch.Tensor, temperatures: torch.Tensor | None = None +) -> torch.Tensor: + """Softmax for sampler logits, optionally with per-row temperatures.""" import torch + if temperatures is not None and logits.is_cuda and torch.version.hip is not None: + from sglang.kernels.ops.sampling.temperature_softmax import ( + temperature_softmax, + ) + + return temperature_softmax(logits, temperatures) + + if temperatures is not None: + logits = logits / temperatures from sglang.kernels.cake_kernels.sampling import supports_softmax if supports_softmax(logits): diff --git a/python/sglang/kernels/ops/sampling/temperature_softmax.py b/python/sglang/kernels/ops/sampling/temperature_softmax.py new file mode 100644 index 0000000000..d057d7b7b0 --- /dev/null +++ b/python/sglang/kernels/ops/sampling/temperature_softmax.py @@ -0,0 +1,170 @@ +"""Split-row temperature softmax for low-row-count sampling workloads.""" + +import torch +import triton +import triton.language as tl + +# Split low-row-count reductions across CTAs and fuse temperature scaling. +_MIN_BLOCK = 2048 +_TARGET_CTAS = 512 +ROW_LIMIT = 32 + + +@triton.jit +def _partial_max_sumexp_kernel( + logits, + temperatures, + partial_max, + partial_sum, + logits_row_stride, + vocab_size, + num_splits, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + split = tl.program_id(1) + t = tl.load(temperatures + row) + offsets = split * BLOCK + tl.arange(0, BLOCK) + mask = offsets < vocab_size + x = tl.load( + logits + row * logits_row_stride + offsets, + mask=mask, + other=-float("inf"), + ) + x = x / t + block_max = tl.max(x, axis=0) + # Fully masked splits contribute zero without evaluating -inf - (-inf). + safe_block_max = tl.where(block_max == -float("inf"), 0.0, block_max) + block_sum = tl.sum(tl.where(mask, tl.exp(x - safe_block_max), 0.0), axis=0) + out_offset = row * num_splits + split + tl.store(partial_max + out_offset, block_max) + tl.store(partial_sum + out_offset, block_sum) + + +@triton.jit +def _combine_kernel( + partial_max, + partial_sum, + row_max, + row_sum, + num_splits, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + offsets = tl.arange(0, BLOCK) + mask = offsets < num_splits + m = tl.load( + partial_max + row * num_splits + offsets, mask=mask, other=-float("inf") + ) + s = tl.load(partial_sum + row * num_splits + offsets, mask=mask, other=0.0) + m_all = tl.max(m, axis=0) + s_all = tl.sum(tl.where(mask, s * tl.exp(m - m_all), 0.0), axis=0) + tl.store(row_max + row, m_all) + tl.store(row_sum + row, s_all) + + +@triton.jit +def _normalize_kernel( + logits, + temperatures, + row_max, + row_sum, + probs, + logits_row_stride, + probs_row_stride, + vocab_size, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + split = tl.program_id(1) + t = tl.load(temperatures + row) + m = tl.load(row_max + row) + s = tl.load(row_sum + row) + offsets = split * BLOCK + tl.arange(0, BLOCK) + mask = offsets < vocab_size + x = tl.load(logits + row * logits_row_stride + offsets, mask=mask, other=0.0) + p = tl.exp(x / t - m) / s + tl.store(probs + row * probs_row_stride + offsets, p, mask=mask) + + +def _split_geometry(rows: int, vocab_size: int): + """Choose a block width and split count near the target grid size.""" + splits = min( + triton.cdiv(_TARGET_CTAS, rows), max(1, triton.cdiv(vocab_size, _MIN_BLOCK)) + ) + block = max(_MIN_BLOCK, triton.next_power_of_2(triton.cdiv(vocab_size, splits))) + return block, triton.cdiv(vocab_size, block) + + +def temperature_softmax( + logits: torch.Tensor, temperatures: torch.Tensor +) -> torch.Tensor: + """Apply temperature softmax, splitting eligible rows across CTAs.""" + if ( + logits.ndim != 2 + or logits.dtype != torch.float32 + or logits.shape[0] == 0 + or logits.shape[1] == 0 + or logits.shape[0] > ROW_LIMIT + or logits.stride(1) != 1 + or not logits.is_cuda + ): + return torch.softmax(logits / temperatures, dim=-1) + + rows, vocab_size = logits.shape + t = temperatures.reshape(-1) + if ( + t.shape[0] != rows + or t.stride(0) != 1 + or t.dtype != logits.dtype + or t.device != logits.device + ): + return torch.softmax(logits / temperatures, dim=-1) + + block, num_splits = _split_geometry(rows, vocab_size) + if num_splits == 1: + return torch.softmax(logits / temperatures, dim=-1) + + probs = torch.empty_like(logits) + partial_max = torch.empty( + (rows, num_splits), dtype=torch.float32, device=logits.device + ) + partial_sum = torch.empty( + (rows, num_splits), dtype=torch.float32, device=logits.device + ) + row_max = torch.empty((rows,), dtype=torch.float32, device=logits.device) + row_sum = torch.empty((rows,), dtype=torch.float32, device=logits.device) + + _partial_max_sumexp_kernel[(rows, num_splits)]( + logits, + t, + partial_max, + partial_sum, + logits.stride(0), + vocab_size, + num_splits, + BLOCK=block, + num_warps=8, + ) + _combine_kernel[(rows,)]( + partial_max, + partial_sum, + row_max, + row_sum, + num_splits, + BLOCK=triton.next_power_of_2(num_splits), + num_warps=1, + ) + _normalize_kernel[(rows, num_splits)]( + logits, + t, + row_max, + row_sum, + probs, + logits.stride(0), + probs.stride(0), + vocab_size, + BLOCK=block, + num_warps=8, + ) + return probs diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 4cde43a1ec..63b689d86e 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -2291,8 +2291,14 @@ def _post_process_topk_ids( ) capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids) recorder_topk_ids = None + _aiter_append = ( + num_fused_shared_experts > 0 + and _use_aiter + and topk_ids.shape[-1] < topk_config.top_k + ) _fold_pad_into_append = False recorder_was_fused = False + _fold_pad_weights_into_append = False if _is_cuda: # LP path: solve LP outside torch.compile (the solver contains an # EP all-reduce that can't run inside compiled regions). @@ -2344,21 +2350,20 @@ def _post_process_topk_ids( # Regression: skipping this mask when EPLB is disabled caused garbage # MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+). # - # Fold: when the fused append+remap kernel runs below (aiter per-rank - # shared-slot path, EPLB off) it folds this padded fill itself - # (pad_fill_id=0 -> remap(0)=0, bit-identical), so skip the separate - # _fill_padded_rows launch here. + # Let append kernels materialize padded ids when remapping is disabled. + eplb_remap_enabled = _eplb_remap_enabled() _fold_pad_into_append = ( - num_fused_shared_experts > 0 - and _use_aiter - and use_per_rank_shared_slots - and not _eplb_remap_enabled() + _aiter_append + and not eplb_remap_enabled + and (use_per_rank_shared_slots or not _skip_hip_pad_mask) + ) + _fold_pad_weights_into_append = ( + _fold_pad_into_append and not use_per_rank_shared_slots ) if not _fold_pad_into_append: _mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0) if ( - _is_hip - and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get() + envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get() and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get() and envs.SGLANG_AITER_MEGA_RANK_SYNC.get() and layer_id is not None @@ -2383,7 +2388,7 @@ def _post_process_topk_ids( # The logical->physical remap is only meaningful when a real # expert-location mapping exists. With a trivial placement and EPLB off # the map is identity so the remap can be skipped safely. - if not recorder_was_fused and _eplb_remap_enabled(): + if not recorder_was_fused and eplb_remap_enabled: topk_ids = topk_ids_logical_to_physical( topk_ids, expert_location_dispatch_info ) @@ -2397,14 +2402,6 @@ def _post_process_topk_ids( topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids ) - # The JIT grouped router and the ROCm decode gate emit the shared slots themselves; - # appending again would write the shared id twice and evict a real routed expert. - _aiter_append = ( - num_fused_shared_experts > 0 - and _use_aiter - and topk_ids.shape[-1] < topk_config.top_k - ) - if _aiter_append and use_per_rank_shared_slots: # Fused path: append shared experts AND apply the per-rank shared-slot # remap in a single Triton kernel. This replaces the original @@ -2447,7 +2444,7 @@ def _post_process_topk_ids( ), ) elif _aiter_append: - M, N = router_logits.shape + N = router_logits.shape[1] scale_factor = ( 1.0 if fused_shared_experts_scaling_factor is None @@ -2465,6 +2462,9 @@ def _post_process_topk_ids( num_fused_shared_experts, scale_factor, N, # base id for shared experts + num_token_non_padded=( + num_token_non_padded if _fold_pad_weights_into_append else None + ), ) elif use_per_rank_shared_slots: @@ -2491,7 +2491,7 @@ def _post_process_topk_ids( fused_shared_experts_scaling_factor ) - if _is_hip and not _skip_hip_pad_mask: + if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append: # Shared-expert append/remap can introduce non-zero weights after the # initial HIP padding mask above. Ensure padded tokens leave this helper # with all expert weights zeroed. diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 50f54c3b74..60defa37c6 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -68,11 +68,44 @@ _is_hip = is_hip() _is_cpu = is_cpu() _is_npu = is_npu() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_is_gfx95 = _is_hip and is_gfx95_supported() if _use_aiter: from aiter.ops.shuffle import shuffle_weight + from aiter.ops.triton.gemm_a16w16 import gemm_a16w16 as _aiter_triton_a16w16 from aiter.tuned_gemm import tgemm +# Route supported decode shapes that are absent from the tuned table. +_A16W16_TRITON_MAX_K = 2048 +_A16W16_TRITON_MAX_M = 64 +_A16W16_TRITON_NARROW_K = 512 +_A16W16_TRITON_NARROW_K_MAX_M = 512 + + +def _prefer_triton_a16w16(x: torch.Tensor, weight: torch.Tensor) -> bool: + """Return whether the Triton BF16 path supports this decode shape.""" + if x.dim() != 2 or weight.dim() != 2: + return False + k = weight.shape[1] + max_m = ( + _A16W16_TRITON_NARROW_K_MAX_M + if k <= _A16W16_TRITON_NARROW_K + else _A16W16_TRITON_MAX_M + ) + return ( + x.is_cuda + and weight.is_cuda + and x.device == weight.device + and x.dtype == torch.bfloat16 + and weight.dtype == torch.bfloat16 + and x.shape[1] == k + and 0 < k <= _A16W16_TRITON_MAX_K + and 0 < x.shape[0] <= max_m + and weight.shape[0] > 0 + and x.is_contiguous() + and weight.is_contiguous() + ) + class Bf16GemmBackend(Enum): AUTO = "auto" @@ -479,6 +512,8 @@ class UnquantizedLinearMethod(LinearMethodBase): return output elif _use_aiter and type(layer.weight.data) is torch.Tensor: + if _is_gfx95 and _prefer_triton_a16w16(x, layer.weight): + return _aiter_triton_a16w16(x, layer.weight, bias, dtype=x.dtype) return tgemm.mm(x, layer.weight, bias, otype=x.dtype) elif ( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py index 21c12c8288..ffe830b3e1 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py @@ -66,7 +66,12 @@ from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.state_capturer.indexer_topk import ( maybe_capture_indexer_topk, ) -from sglang.srt.utils import BumpAllocator, get_bool_env_var +from sglang.srt.utils import ( + BumpAllocator, + cdiv, + get_bool_env_var, + get_device_core_count, +) logger = logging.getLogger(__name__) _SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get() @@ -145,6 +150,39 @@ if _use_aiter_gfx95: ) +_ABSORB_BMM_DECODE_MAX_M = 256 +_absorb_bmm_num_cu = 0 + + +def _absorb_bmm_config(heads: int, m: int, n: int) -> Optional[dict]: + """Return decode tile settings, or None outside the decode row limit.""" + global _absorb_bmm_num_cu + if m <= 0 or m > _ABSORB_BMM_DECODE_MAX_M: + return None + if not _absorb_bmm_num_cu: + _absorb_bmm_num_cu = get_device_core_count() + num_cu = _absorb_bmm_num_cu + m_tiles_16 = cdiv(m, 16) + block_n = 128 + while block_n > 32 and 4 * heads * m_tiles_16 * cdiv(n, block_n) < 3 * num_cu: + block_n //= 2 + n_tiles = cdiv(n, block_n) + block_m = 16 + while block_m < 64 and heads * cdiv(m, block_m) * n_tiles > 2 * num_cu: + block_m *= 2 + workgroups = heads * cdiv(m, block_m) * n_tiles + return { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": max(1, cdiv(workgroups, num_cu)), + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + } + + def _absorb_weight_bf16(w: torch.Tensor, w_scale) -> torch.Tensor: """Dequantize an absorbed MLA weight, skipping the pass when it is a no-op.""" if ( @@ -196,6 +234,9 @@ def rocm_absorb_q_bmm( transpose_bm=False, # (B, M, N) transpose_bm_in=True, # (M, B, K) dtype=torch.bfloat16, + config=_absorb_bmm_config( + q_nope.shape[1], q_nope.shape[0], attn.w_kc.shape[2] + ), ) ) else: @@ -254,6 +295,9 @@ def rocm_absorb_v_bmm( transpose_bm=True, transpose_bm_in=True, dtype=torch.bfloat16, + config=_absorb_bmm_config( + attn.num_local_heads, attn_output.shape[0], attn.w_vc.shape[-1] + ), ) elif not is_in_tc_piecewise_cuda_graph(): # Same (batch, heads, dim) layout as the quantized paths above, so the diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index a312a8b403..c1bacff0e1 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple import torch +from sglang.kernels.ops.sampling import softmax as sampling_softmax from sglang.kernels.ops.speculative.spec_tree import ( sgl_build_tree_kernel_efficient_triton, verify_tree_greedy_kernel_triton, @@ -743,8 +744,6 @@ def eagle_sample( Verify and find accepted tokens based on logits output and batch (which contains spec decoding information). """ - import torch.nn.functional as F - from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -927,8 +926,8 @@ def eagle_sample( sampling_info.temperatures, verify_input.draft_token_num, dim=0 ) # (bs * num_draft_tokens, 1) - target_probs = F.softmax( - next_token_logits / expanded_temperature, dim=-1 + target_probs = sampling_softmax( + next_token_logits, temperatures=expanded_temperature ) # (bs * num_draft_tokens, vocab_size) maybe_detect_nan(target_probs, "v2 verify: target_probs after softmax") if sampling_info.need_top_k_sampling: @@ -1054,8 +1053,6 @@ def eagle_sample( def eagle_prepare_for_decode(batch: ScheduleBatch): batch.maybe_evict_swa() - bs = batch.batch_size() - # Accumulate penalty # This is a relaxed version of penalties for speculative decoding. if batch.sampling_info.penalizer_orchestrator.is_required: diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 1c5390b00f..e39826167f 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Tuple import torch from huggingface_hub import snapshot_download +from sglang.kernels.ops.sampling import softmax as sampling_softmax from sglang.kernels.ops.speculative.cache_locs import ( align_evict_mask_to_page_size as align_evict_mask_to_page_size, ) @@ -31,6 +32,7 @@ from sglang.kernels.ops.speculative.cache_locs import ( from sglang.kernels.ops.speculative.eagle import ( fill_accept_out_cache_loc_func as fill_accept_out_cache_loc_func, ) +from sglang.kernels.ops.speculative.row_argmax import row_argmax from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.constrained.base_grammar_backend import GrammarMask @@ -138,7 +140,7 @@ def fast_sample(probs: torch.Tensor, num_samples: int = 1): q.clamp_min_(torch.finfo(torch.float32).tiny) scores = probs.float() / q if num_samples == 1: - sample_index = scores.argmax(dim=-1, keepdim=True) + sample_index = row_argmax(scores).unsqueeze(-1) else: sample_index = scores.topk(num_samples, dim=-1).indices sample_p = probs.gather(1, sample_index) @@ -157,8 +159,8 @@ def renorm_draft_probs( temperature (higher acceptance; correctness holds for any q). """ if not use_rejection_sampling or not next_token_logits.size(0): - return torch.softmax(next_token_logits, dim=-1) - return torch.softmax(next_token_logits / sampling_info.temperatures, dim=-1) + return sampling_softmax(next_token_logits) + return sampling_softmax(next_token_logits, temperatures=sampling_info.temperatures) def sample_draft_proposal( @@ -186,7 +188,7 @@ def sample_draft_proposal( (p - q)+ it resamples from is p itself. Both arms commit the target argmax, which is what greedy means. Drop that renorm and this stops holding. """ - probs = torch.softmax(next_token_logits / temperatures, dim=-1) + probs = sampling_softmax(next_token_logits, temperatures=temperatures) topk_p, topk_index = fast_sample(probs, num_samples=1) if top_ks is not None: # Assert rather than skip on a device mismatch: a host-side top_ks would diff --git a/test/registered/kernels/ops/sampling/test_temperature_softmax.py b/test/registered/kernels/ops/sampling/test_temperature_softmax.py new file mode 100644 index 0000000000..876ee1df43 --- /dev/null +++ b/test/registered/kernels/ops/sampling/test_temperature_softmax.py @@ -0,0 +1,75 @@ +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-small") +register_amd_ci(est_time=10, stage="stage-b", runner_config="1-gpu-small-amd") + +import unittest + +import torch + +from sglang.kernels.ops.sampling import softmax +from sglang.test.test_utils import CustomTestCase + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required for this test.") +class TestTemperatureSoftmax(CustomTestCase): + @staticmethod + def temperatures(rows: int) -> torch.Tensor: + return torch.linspace( + 0.5, 1.5, rows, dtype=torch.float32, device="cuda" + ).unsqueeze(1) + + def assert_matches_torch(self, logits: torch.Tensor, temperatures: torch.Tensor): + expected = torch.softmax(logits / temperatures, dim=-1) + actual = softmax(logits, temperatures=temperatures) + + torch.testing.assert_close(actual, expected, rtol=2e-5, atol=1e-7) + + def test_dense_logits(self): + generator = torch.Generator(device="cuda").manual_seed(0) + for rows, vocab_size in ( + (0, 1024), + (1, 0), + (1, 1024), + (1, 32000), + (7, 65537), + (32, 128257), + (33, 4096), + ): + with self.subTest(rows=rows, vocab_size=vocab_size): + logits = torch.randn( + (rows, vocab_size), + dtype=torch.float32, + device="cuda", + generator=generator, + ) + self.assert_matches_torch(logits, self.temperatures(rows)) + + def test_noncontiguous_logits_fall_back(self): + logits = torch.randn((4096, 3), dtype=torch.float32, device="cuda").T + self.assert_matches_torch(logits, self.temperatures(logits.shape[0])) + + def test_sparse_grammar_mask_with_fully_masked_splits(self): + vocab_size = 65537 + allowed = torch.tensor( + [0, vocab_size // 2, vocab_size - 1], device="cuda", dtype=torch.long + ) + + for rows in (1, 3, 16, 32): + with self.subTest(rows=rows): + logits = torch.full( + (rows, vocab_size), + -float("inf"), + dtype=torch.float32, + device="cuda", + ) + values = torch.arange( + rows * allowed.numel(), dtype=torch.float32, device="cuda" + ).reshape(rows, allowed.numel()) + values = values.remainder(7).sub_(3) + logits[:, allowed] = values + self.assert_matches_torch(logits, self.temperatures(rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/moe/test_fused_append_shared_experts_top6.py b/test/registered/moe/test_fused_append_shared_experts_top6.py index 47393b6915..f94c323cd2 100644 --- a/test/registered/moe/test_fused_append_shared_experts_top6.py +++ b/test/registered/moe/test_fused_append_shared_experts_top6.py @@ -101,6 +101,31 @@ class TestFusedAppendSharedExpertsTop6(CustomTestCase): self.assertTrue(torch.equal(got_ids[:, :k], topk_ids)) self.assertTrue(torch.allclose(got_w[:, :k], topk_weights)) + def test_padded_rows(self): + m, k, s = 8, 6, 2 + topk_ids, topk_weights = self._make_inputs(m, k) + for valid_rows in (0, 3, m): + with self.subTest(valid_rows=valid_rows): + num_token_non_padded = torch.tensor( + [valid_rows], dtype=torch.int32, device=topk_ids.device + ) + got_ids, got_w = fused_append_shared_experts( + topk_ids, + topk_weights, + s, + 0.5, + N=self.N_BASE, + num_token_non_padded=num_token_non_padded, + ) + exp_ids, exp_w = _reference_append( + topk_ids, topk_weights, s, 0.5, self.N_BASE + ) + exp_ids[valid_rows:, :k] = 0 + exp_w[valid_rows:] = 0 + + self.assertTrue(torch.equal(got_ids, exp_ids)) + self.assertTrue(torch.equal(got_w, exp_w)) + def test_no_shared_experts_is_noop(self): """s == 0 returns the inputs untouched (no kernel launch).""" topk_ids, topk_weights = self._make_inputs(4, 6) diff --git a/test/registered/moe/test_topk_padded_region.py b/test/registered/moe/test_topk_padded_region.py index 929c96d7a6..d3fc006a64 100644 --- a/test/registered/moe/test_topk_padded_region.py +++ b/test/registered/moe/test_topk_padded_region.py @@ -189,6 +189,33 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase): finally: topk_mod._skip_hip_pad_mask = orig + def test_flag_skips_weight_masking_with_shared_append(self): + n, k, n_valid = 16, 8, 5 + topk_weights = torch.rand((n, k), device=self.DEVICE) + 0.5 + topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32) + router_logits = torch.rand((n, 64), device=self.DEVICE) + pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32) + cfg = TopKConfig(top_k=k + 1, num_fused_shared_experts=1) + orig_skip = topk_mod._skip_hip_pad_mask + orig_aiter = topk_mod._use_aiter + topk_mod._skip_hip_pad_mask = True + topk_mod._use_aiter = True + try: + out_ids, out_weights, _ = _post_process_topk_ids( + topk_ids, + topk_weights, + cfg, + router_logits, + layer_id=0, + num_token_non_padded=pad, + ) + finally: + topk_mod._skip_hip_pad_mask = orig_skip + topk_mod._use_aiter = orig_aiter + + self.assertTrue(torch.all(out_ids[n_valid:, :k] == 0)) + self.assertTrue(torch.all(out_weights[n_valid:] > 0)) + @unittest.skipUnless(torch.cuda.is_available(), "padded-region masking needs a GPU") @unittest.skipIf(_IS_HIP, "HIP keeps the pre-mask routing contract (AITER/MORI)") diff --git a/test/registered/unit/layers/quantization/test_aiter_a16w16_gate.py b/test/registered/unit/layers/quantization/test_aiter_a16w16_gate.py new file mode 100644 index 0000000000..7ce6d90a00 --- /dev/null +++ b/test/registered/unit/layers/quantization/test_aiter_a16w16_gate.py @@ -0,0 +1,46 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.quantization.unquant import _prefer_triton_a16w16 +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _tensor(rows, cols, *, dtype=torch.bfloat16, device="cuda:0", contiguous=True): + return SimpleNamespace( + shape=(rows, cols), + dtype=dtype, + device=device, + is_cuda=device.startswith("cuda"), + dim=lambda: 2, + is_contiguous=lambda: contiguous, + ) + + +class TestAiterA16W16Gate(CustomTestCase): + def test_supported_decode_shapes(self): + for rows, k in ((1, 2048), (64, 2048), (512, 512)): + with self.subTest(rows=rows, k=k): + self.assertTrue(_prefer_triton_a16w16(_tensor(rows, k), _tensor(16, k))) + + def test_unsupported_shapes(self): + cases = ( + (_tensor(65, 2048), _tensor(16, 2048)), + (_tensor(513, 512), _tensor(16, 512)), + (_tensor(1, 2049), _tensor(16, 2049)), + (_tensor(1, 2048), _tensor(16, 1024)), + (_tensor(0, 2048), _tensor(16, 2048)), + (_tensor(1, 2048, device="cpu"), _tensor(16, 2048, device="cpu")), + (_tensor(1, 2048, contiguous=False), _tensor(16, 2048)), + ) + for x, weight in cases: + with self.subTest(x=x.shape, weight=weight.shape): + self.assertFalse(_prefer_triton_a16w16(x, weight)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py index 71325cd056..fee491d922 100644 --- a/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py +++ b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py @@ -23,6 +23,7 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") _CAN_FUSE = forward_mla_rocm._can_fuse_bmm_rope_cat_and_cache +_ABSORB_CONFIG = forward_mla_rocm._absorb_bmm_config def _attn(**overrides): @@ -90,5 +91,39 @@ class TestFusedAbsorbGate(CustomTestCase): self.assertFalse(_CAN_FUSE(_attn(**{name: broken}))) +class TestAbsorbBmmConfig(CustomTestCase): + def setUp(self): + self._saved = forward_mla_rocm._absorb_bmm_num_cu + forward_mla_rocm._absorb_bmm_num_cu = 256 + self.addCleanup( + setattr, + forward_mla_rocm, + "_absorb_bmm_num_cu", + self._saved, + ) + + def test_decode_tiles(self): + cases = ( + ((16, 1, 512), (16, 32, 1)), + ((16, 48, 512), (16, 128, 1)), + ((16, 200, 512), (32, 128, 2)), + ) + for args, expected in cases: + with self.subTest(args=args): + config = _ABSORB_CONFIG(*args) + self.assertEqual( + ( + config["BLOCK_SIZE_M"], + config["BLOCK_SIZE_N"], + config["waves_per_eu"], + ), + expected, + ) + + def test_large_row_count_falls_back(self): + self.assertIsNone(_ABSORB_CONFIG(16, 0, 512)) + self.assertIsNone(_ABSORB_CONFIG(16, 257, 512)) + + if __name__ == "__main__": unittest.main()