[ROCm] GLM-5.2 decode path: decode-shaped MoE/MLA tiles, split speculative softmax, and bf16 GEMM routing (#41725)

Co-authored-by: kyle-256 <Kyle.Zhao@amd.com>
This commit is contained in:
Zhang, Jiejing
2026-10-04 11:19:37 -07:00
committed by GitHub
co-authored by kyle-256
parent 7a719e9a65
commit aa0ce20201
13 changed files with 524 additions and 38 deletions
@@ -1288,10 +1288,12 @@ def _fused_append_shared_experts_kernel(
out_weights_ptr,
N_BASE, # runtime scalar
scale_factor, # runtime scalar
num_token_non_padded_ptr, # 1-elem int tensor; only read when HAS_PADDING
K: tl.constexpr,
S: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_S: tl.constexpr,
HAS_PADDING: tl.constexpr,
):
"""
for m in range(M):
@@ -1317,22 +1319,35 @@ def _fused_append_shared_experts_kernel(
ids = tl.load(topk_ids_ptr + ids_row_ptr + offs_k, mask=mask_k)
ws = tl.load(topk_weights_ptr + w_row_ptr + offs_k, mask=mask_k)
tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k)
tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k)
offs_s = tl.arange(0, BLOCK_S)
mask_s = offs_s < S
shared_ids = tl.cast(N_BASE + offs_s, ids.dtype)
shared_ws = tl.full([BLOCK_S], scale_factor, dtype=ws.dtype)
if HAS_PADDING:
# Padded rows zero routed ids and all weights.
if pid >= tl.load(num_token_non_padded_ptr):
ids = tl.zeros([BLOCK_K], dtype=ids.dtype)
ws = tl.zeros([BLOCK_K], dtype=ws.dtype)
shared_ws = tl.zeros([BLOCK_S], dtype=ws.dtype)
tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k)
tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k)
tl.store(out_ids_ptr + out_ids_row_ptr + K + offs_s, shared_ids, mask=mask_s)
tl.store(out_weights_ptr + out_w_row_ptr + K + offs_s, shared_ws, mask=mask_s)
def fused_append_shared_experts(
topk_ids, topk_weights, num_fused_shared_experts, scale_factor, N=None
topk_ids,
topk_weights,
num_fused_shared_experts,
scale_factor,
N=None,
num_token_non_padded=None,
):
"""Append shared experts and optionally materialize padded rows."""
assert N is not None, "N (shared expert base id) must be provided"
m, k = topk_ids.shape
s = int(num_fused_shared_experts)
@@ -1344,6 +1359,8 @@ def fused_append_shared_experts(
(m, k + s), dtype=topk_weights.dtype, device=topk_weights.device
)
has_padding = num_token_non_padded is not None
ntnp_ptr = num_token_non_padded if has_padding else topk_ids
_fused_append_shared_experts_kernel[(m,)](
topk_ids,
topk_weights,
@@ -1351,10 +1368,12 @@ def fused_append_shared_experts(
out_weights,
N_BASE=N,
scale_factor=scale_factor,
num_token_non_padded_ptr=ntnp_ptr,
K=k,
S=s,
BLOCK_K=triton.next_power_of_2(k),
BLOCK_S=triton.next_power_of_2(s),
HAS_PADDING=has_padding,
num_warps=1,
)
return out_ids, out_weights
+13 -2
View File
@@ -40,10 +40,21 @@ def cake_softmax(logits: torch.Tensor) -> torch.Tensor:
return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits)
def softmax(logits: torch.Tensor) -> torch.Tensor:
"""Softmax for sampler logits, with a qualified Blackwell fast path."""
def softmax(
logits: torch.Tensor, temperatures: torch.Tensor | None = None
) -> torch.Tensor:
"""Softmax for sampler logits, optionally with per-row temperatures."""
import torch
if temperatures is not None and logits.is_cuda and torch.version.hip is not None:
from sglang.kernels.ops.sampling.temperature_softmax import (
temperature_softmax,
)
return temperature_softmax(logits, temperatures)
if temperatures is not None:
logits = logits / temperatures
from sglang.kernels.cake_kernels.sampling import supports_softmax
if supports_softmax(logits):
@@ -0,0 +1,170 @@
"""Split-row temperature softmax for low-row-count sampling workloads."""
import torch
import triton
import triton.language as tl
# Split low-row-count reductions across CTAs and fuse temperature scaling.
_MIN_BLOCK = 2048
_TARGET_CTAS = 512
ROW_LIMIT = 32
@triton.jit
def _partial_max_sumexp_kernel(
logits,
temperatures,
partial_max,
partial_sum,
logits_row_stride,
vocab_size,
num_splits,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
split = tl.program_id(1)
t = tl.load(temperatures + row)
offsets = split * BLOCK + tl.arange(0, BLOCK)
mask = offsets < vocab_size
x = tl.load(
logits + row * logits_row_stride + offsets,
mask=mask,
other=-float("inf"),
)
x = x / t
block_max = tl.max(x, axis=0)
# Fully masked splits contribute zero without evaluating -inf - (-inf).
safe_block_max = tl.where(block_max == -float("inf"), 0.0, block_max)
block_sum = tl.sum(tl.where(mask, tl.exp(x - safe_block_max), 0.0), axis=0)
out_offset = row * num_splits + split
tl.store(partial_max + out_offset, block_max)
tl.store(partial_sum + out_offset, block_sum)
@triton.jit
def _combine_kernel(
partial_max,
partial_sum,
row_max,
row_sum,
num_splits,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
offsets = tl.arange(0, BLOCK)
mask = offsets < num_splits
m = tl.load(
partial_max + row * num_splits + offsets, mask=mask, other=-float("inf")
)
s = tl.load(partial_sum + row * num_splits + offsets, mask=mask, other=0.0)
m_all = tl.max(m, axis=0)
s_all = tl.sum(tl.where(mask, s * tl.exp(m - m_all), 0.0), axis=0)
tl.store(row_max + row, m_all)
tl.store(row_sum + row, s_all)
@triton.jit
def _normalize_kernel(
logits,
temperatures,
row_max,
row_sum,
probs,
logits_row_stride,
probs_row_stride,
vocab_size,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
split = tl.program_id(1)
t = tl.load(temperatures + row)
m = tl.load(row_max + row)
s = tl.load(row_sum + row)
offsets = split * BLOCK + tl.arange(0, BLOCK)
mask = offsets < vocab_size
x = tl.load(logits + row * logits_row_stride + offsets, mask=mask, other=0.0)
p = tl.exp(x / t - m) / s
tl.store(probs + row * probs_row_stride + offsets, p, mask=mask)
def _split_geometry(rows: int, vocab_size: int):
"""Choose a block width and split count near the target grid size."""
splits = min(
triton.cdiv(_TARGET_CTAS, rows), max(1, triton.cdiv(vocab_size, _MIN_BLOCK))
)
block = max(_MIN_BLOCK, triton.next_power_of_2(triton.cdiv(vocab_size, splits)))
return block, triton.cdiv(vocab_size, block)
def temperature_softmax(
logits: torch.Tensor, temperatures: torch.Tensor
) -> torch.Tensor:
"""Apply temperature softmax, splitting eligible rows across CTAs."""
if (
logits.ndim != 2
or logits.dtype != torch.float32
or logits.shape[0] == 0
or logits.shape[1] == 0
or logits.shape[0] > ROW_LIMIT
or logits.stride(1) != 1
or not logits.is_cuda
):
return torch.softmax(logits / temperatures, dim=-1)
rows, vocab_size = logits.shape
t = temperatures.reshape(-1)
if (
t.shape[0] != rows
or t.stride(0) != 1
or t.dtype != logits.dtype
or t.device != logits.device
):
return torch.softmax(logits / temperatures, dim=-1)
block, num_splits = _split_geometry(rows, vocab_size)
if num_splits == 1:
return torch.softmax(logits / temperatures, dim=-1)
probs = torch.empty_like(logits)
partial_max = torch.empty(
(rows, num_splits), dtype=torch.float32, device=logits.device
)
partial_sum = torch.empty(
(rows, num_splits), dtype=torch.float32, device=logits.device
)
row_max = torch.empty((rows,), dtype=torch.float32, device=logits.device)
row_sum = torch.empty((rows,), dtype=torch.float32, device=logits.device)
_partial_max_sumexp_kernel[(rows, num_splits)](
logits,
t,
partial_max,
partial_sum,
logits.stride(0),
vocab_size,
num_splits,
BLOCK=block,
num_warps=8,
)
_combine_kernel[(rows,)](
partial_max,
partial_sum,
row_max,
row_sum,
num_splits,
BLOCK=triton.next_power_of_2(num_splits),
num_warps=1,
)
_normalize_kernel[(rows, num_splits)](
logits,
t,
row_max,
row_sum,
probs,
logits.stride(0),
probs.stride(0),
vocab_size,
BLOCK=block,
num_warps=8,
)
return probs
+21 -21
View File
@@ -2291,8 +2291,14 @@ def _post_process_topk_ids(
)
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
recorder_topk_ids = None
_aiter_append = (
num_fused_shared_experts > 0
and _use_aiter
and topk_ids.shape[-1] < topk_config.top_k
)
_fold_pad_into_append = False
recorder_was_fused = False
_fold_pad_weights_into_append = False
if _is_cuda:
# LP path: solve LP outside torch.compile (the solver contains an
# EP all-reduce that can't run inside compiled regions).
@@ -2344,21 +2350,20 @@ def _post_process_topk_ids(
# Regression: skipping this mask when EPLB is disabled caused garbage
# MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+).
#
# Fold: when the fused append+remap kernel runs below (aiter per-rank
# shared-slot path, EPLB off) it folds this padded fill itself
# (pad_fill_id=0 -> remap(0)=0, bit-identical), so skip the separate
# _fill_padded_rows launch here.
# Let append kernels materialize padded ids when remapping is disabled.
eplb_remap_enabled = _eplb_remap_enabled()
_fold_pad_into_append = (
num_fused_shared_experts > 0
and _use_aiter
and use_per_rank_shared_slots
and not _eplb_remap_enabled()
_aiter_append
and not eplb_remap_enabled
and (use_per_rank_shared_slots or not _skip_hip_pad_mask)
)
_fold_pad_weights_into_append = (
_fold_pad_into_append and not use_per_rank_shared_slots
)
if not _fold_pad_into_append:
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0)
if (
_is_hip
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get()
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
and layer_id is not None
@@ -2383,7 +2388,7 @@ def _post_process_topk_ids(
# The logical->physical remap is only meaningful when a real
# expert-location mapping exists. With a trivial placement and EPLB off
# the map is identity so the remap can be skipped safely.
if not recorder_was_fused and _eplb_remap_enabled():
if not recorder_was_fused and eplb_remap_enabled:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info
)
@@ -2397,14 +2402,6 @@ def _post_process_topk_ids(
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
)
# The JIT grouped router and the ROCm decode gate emit the shared slots themselves;
# appending again would write the shared id twice and evict a real routed expert.
_aiter_append = (
num_fused_shared_experts > 0
and _use_aiter
and topk_ids.shape[-1] < topk_config.top_k
)
if _aiter_append and use_per_rank_shared_slots:
# Fused path: append shared experts AND apply the per-rank shared-slot
# remap in a single Triton kernel. This replaces the original
@@ -2447,7 +2444,7 @@ def _post_process_topk_ids(
),
)
elif _aiter_append:
M, N = router_logits.shape
N = router_logits.shape[1]
scale_factor = (
1.0
if fused_shared_experts_scaling_factor is None
@@ -2465,6 +2462,9 @@ def _post_process_topk_ids(
num_fused_shared_experts,
scale_factor,
N, # base id for shared experts
num_token_non_padded=(
num_token_non_padded if _fold_pad_weights_into_append else None
),
)
elif use_per_rank_shared_slots:
@@ -2491,7 +2491,7 @@ def _post_process_topk_ids(
fused_shared_experts_scaling_factor
)
if _is_hip and not _skip_hip_pad_mask:
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
# Shared-expert append/remap can introduce non-zero weights after the
# initial HIP padding mask above. Ensure padded tokens leave this helper
# with all expert weights zeroed.
@@ -68,11 +68,44 @@ _is_hip = is_hip()
_is_cpu = is_cpu()
_is_npu = is_npu()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
_is_gfx95 = _is_hip and is_gfx95_supported()
if _use_aiter:
from aiter.ops.shuffle import shuffle_weight
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16 as _aiter_triton_a16w16
from aiter.tuned_gemm import tgemm
# Route supported decode shapes that are absent from the tuned table.
_A16W16_TRITON_MAX_K = 2048
_A16W16_TRITON_MAX_M = 64
_A16W16_TRITON_NARROW_K = 512
_A16W16_TRITON_NARROW_K_MAX_M = 512
def _prefer_triton_a16w16(x: torch.Tensor, weight: torch.Tensor) -> bool:
"""Return whether the Triton BF16 path supports this decode shape."""
if x.dim() != 2 or weight.dim() != 2:
return False
k = weight.shape[1]
max_m = (
_A16W16_TRITON_NARROW_K_MAX_M
if k <= _A16W16_TRITON_NARROW_K
else _A16W16_TRITON_MAX_M
)
return (
x.is_cuda
and weight.is_cuda
and x.device == weight.device
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and x.shape[1] == k
and 0 < k <= _A16W16_TRITON_MAX_K
and 0 < x.shape[0] <= max_m
and weight.shape[0] > 0
and x.is_contiguous()
and weight.is_contiguous()
)
class Bf16GemmBackend(Enum):
AUTO = "auto"
@@ -479,6 +512,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
return output
elif _use_aiter and type(layer.weight.data) is torch.Tensor:
if _is_gfx95 and _prefer_triton_a16w16(x, layer.weight):
return _aiter_triton_a16w16(x, layer.weight, bias, dtype=x.dtype)
return tgemm.mm(x, layer.weight, bias, otype=x.dtype)
elif (
@@ -66,7 +66,12 @@ from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.state_capturer.indexer_topk import (
maybe_capture_indexer_topk,
)
from sglang.srt.utils import BumpAllocator, get_bool_env_var
from sglang.srt.utils import (
BumpAllocator,
cdiv,
get_bool_env_var,
get_device_core_count,
)
logger = logging.getLogger(__name__)
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
@@ -145,6 +150,39 @@ if _use_aiter_gfx95:
)
_ABSORB_BMM_DECODE_MAX_M = 256
_absorb_bmm_num_cu = 0
def _absorb_bmm_config(heads: int, m: int, n: int) -> Optional[dict]:
"""Return decode tile settings, or None outside the decode row limit."""
global _absorb_bmm_num_cu
if m <= 0 or m > _ABSORB_BMM_DECODE_MAX_M:
return None
if not _absorb_bmm_num_cu:
_absorb_bmm_num_cu = get_device_core_count()
num_cu = _absorb_bmm_num_cu
m_tiles_16 = cdiv(m, 16)
block_n = 128
while block_n > 32 and 4 * heads * m_tiles_16 * cdiv(n, block_n) < 3 * num_cu:
block_n //= 2
n_tiles = cdiv(n, block_n)
block_m = 16
while block_m < 64 and heads * cdiv(m, block_m) * n_tiles > 2 * num_cu:
block_m *= 2
workgroups = heads * cdiv(m, block_m) * n_tiles
return {
"BLOCK_SIZE_M": block_m,
"BLOCK_SIZE_N": block_n,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 2,
"waves_per_eu": max(1, cdiv(workgroups, num_cu)),
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
}
def _absorb_weight_bf16(w: torch.Tensor, w_scale) -> torch.Tensor:
"""Dequantize an absorbed MLA weight, skipping the pass when it is a no-op."""
if (
@@ -196,6 +234,9 @@ def rocm_absorb_q_bmm(
transpose_bm=False, # (B, M, N)
transpose_bm_in=True, # (M, B, K)
dtype=torch.bfloat16,
config=_absorb_bmm_config(
q_nope.shape[1], q_nope.shape[0], attn.w_kc.shape[2]
),
)
)
else:
@@ -254,6 +295,9 @@ def rocm_absorb_v_bmm(
transpose_bm=True,
transpose_bm_in=True,
dtype=torch.bfloat16,
config=_absorb_bmm_config(
attn.num_local_heads, attn_output.shape[0], attn.w_vc.shape[-1]
),
)
elif not is_in_tc_piecewise_cuda_graph():
# Same (batch, heads, dim) layout as the quantized paths above, so the
+3 -6
View File
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
from sglang.kernels.ops.sampling import softmax as sampling_softmax
from sglang.kernels.ops.speculative.spec_tree import (
sgl_build_tree_kernel_efficient_triton,
verify_tree_greedy_kernel_triton,
@@ -743,8 +744,6 @@ def eagle_sample(
Verify and find accepted tokens based on logits output and batch
(which contains spec decoding information).
"""
import torch.nn.functional as F
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -927,8 +926,8 @@ def eagle_sample(
sampling_info.temperatures, verify_input.draft_token_num, dim=0
) # (bs * num_draft_tokens, 1)
target_probs = F.softmax(
next_token_logits / expanded_temperature, dim=-1
target_probs = sampling_softmax(
next_token_logits, temperatures=expanded_temperature
) # (bs * num_draft_tokens, vocab_size)
maybe_detect_nan(target_probs, "v2 verify: target_probs after softmax")
if sampling_info.need_top_k_sampling:
@@ -1054,8 +1053,6 @@ def eagle_sample(
def eagle_prepare_for_decode(batch: ScheduleBatch):
batch.maybe_evict_swa()
bs = batch.batch_size()
# Accumulate penalty
# This is a relaxed version of penalties for speculative decoding.
if batch.sampling_info.penalizer_orchestrator.is_required:
+6 -4
View File
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Tuple
import torch
from huggingface_hub import snapshot_download
from sglang.kernels.ops.sampling import softmax as sampling_softmax
from sglang.kernels.ops.speculative.cache_locs import (
align_evict_mask_to_page_size as align_evict_mask_to_page_size,
)
@@ -31,6 +32,7 @@ from sglang.kernels.ops.speculative.cache_locs import (
from sglang.kernels.ops.speculative.eagle import (
fill_accept_out_cache_loc_func as fill_accept_out_cache_loc_func,
)
from sglang.kernels.ops.speculative.row_argmax import row_argmax
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.constrained.base_grammar_backend import GrammarMask
@@ -138,7 +140,7 @@ def fast_sample(probs: torch.Tensor, num_samples: int = 1):
q.clamp_min_(torch.finfo(torch.float32).tiny)
scores = probs.float() / q
if num_samples == 1:
sample_index = scores.argmax(dim=-1, keepdim=True)
sample_index = row_argmax(scores).unsqueeze(-1)
else:
sample_index = scores.topk(num_samples, dim=-1).indices
sample_p = probs.gather(1, sample_index)
@@ -157,8 +159,8 @@ def renorm_draft_probs(
temperature (higher acceptance; correctness holds for any q).
"""
if not use_rejection_sampling or not next_token_logits.size(0):
return torch.softmax(next_token_logits, dim=-1)
return torch.softmax(next_token_logits / sampling_info.temperatures, dim=-1)
return sampling_softmax(next_token_logits)
return sampling_softmax(next_token_logits, temperatures=sampling_info.temperatures)
def sample_draft_proposal(
@@ -186,7 +188,7 @@ def sample_draft_proposal(
(p - q)+ it resamples from is p itself. Both arms commit the target argmax,
which is what greedy means. Drop that renorm and this stops holding.
"""
probs = torch.softmax(next_token_logits / temperatures, dim=-1)
probs = sampling_softmax(next_token_logits, temperatures=temperatures)
topk_p, topk_index = fast_sample(probs, num_samples=1)
if top_ks is not None:
# Assert rather than skip on a device mismatch: a host-side top_ks would
@@ -0,0 +1,75 @@
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-small")
register_amd_ci(est_time=10, stage="stage-b", runner_config="1-gpu-small-amd")
import unittest
import torch
from sglang.kernels.ops.sampling import softmax
from sglang.test.test_utils import CustomTestCase
@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required for this test.")
class TestTemperatureSoftmax(CustomTestCase):
@staticmethod
def temperatures(rows: int) -> torch.Tensor:
return torch.linspace(
0.5, 1.5, rows, dtype=torch.float32, device="cuda"
).unsqueeze(1)
def assert_matches_torch(self, logits: torch.Tensor, temperatures: torch.Tensor):
expected = torch.softmax(logits / temperatures, dim=-1)
actual = softmax(logits, temperatures=temperatures)
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=1e-7)
def test_dense_logits(self):
generator = torch.Generator(device="cuda").manual_seed(0)
for rows, vocab_size in (
(0, 1024),
(1, 0),
(1, 1024),
(1, 32000),
(7, 65537),
(32, 128257),
(33, 4096),
):
with self.subTest(rows=rows, vocab_size=vocab_size):
logits = torch.randn(
(rows, vocab_size),
dtype=torch.float32,
device="cuda",
generator=generator,
)
self.assert_matches_torch(logits, self.temperatures(rows))
def test_noncontiguous_logits_fall_back(self):
logits = torch.randn((4096, 3), dtype=torch.float32, device="cuda").T
self.assert_matches_torch(logits, self.temperatures(logits.shape[0]))
def test_sparse_grammar_mask_with_fully_masked_splits(self):
vocab_size = 65537
allowed = torch.tensor(
[0, vocab_size // 2, vocab_size - 1], device="cuda", dtype=torch.long
)
for rows in (1, 3, 16, 32):
with self.subTest(rows=rows):
logits = torch.full(
(rows, vocab_size),
-float("inf"),
dtype=torch.float32,
device="cuda",
)
values = torch.arange(
rows * allowed.numel(), dtype=torch.float32, device="cuda"
).reshape(rows, allowed.numel())
values = values.remainder(7).sub_(3)
logits[:, allowed] = values
self.assert_matches_torch(logits, self.temperatures(rows))
if __name__ == "__main__":
unittest.main()
@@ -101,6 +101,31 @@ class TestFusedAppendSharedExpertsTop6(CustomTestCase):
self.assertTrue(torch.equal(got_ids[:, :k], topk_ids))
self.assertTrue(torch.allclose(got_w[:, :k], topk_weights))
def test_padded_rows(self):
m, k, s = 8, 6, 2
topk_ids, topk_weights = self._make_inputs(m, k)
for valid_rows in (0, 3, m):
with self.subTest(valid_rows=valid_rows):
num_token_non_padded = torch.tensor(
[valid_rows], dtype=torch.int32, device=topk_ids.device
)
got_ids, got_w = fused_append_shared_experts(
topk_ids,
topk_weights,
s,
0.5,
N=self.N_BASE,
num_token_non_padded=num_token_non_padded,
)
exp_ids, exp_w = _reference_append(
topk_ids, topk_weights, s, 0.5, self.N_BASE
)
exp_ids[valid_rows:, :k] = 0
exp_w[valid_rows:] = 0
self.assertTrue(torch.equal(got_ids, exp_ids))
self.assertTrue(torch.equal(got_w, exp_w))
def test_no_shared_experts_is_noop(self):
"""s == 0 returns the inputs untouched (no kernel launch)."""
topk_ids, topk_weights = self._make_inputs(4, 6)
@@ -189,6 +189,33 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase):
finally:
topk_mod._skip_hip_pad_mask = orig
def test_flag_skips_weight_masking_with_shared_append(self):
n, k, n_valid = 16, 8, 5
topk_weights = torch.rand((n, k), device=self.DEVICE) + 0.5
topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32)
router_logits = torch.rand((n, 64), device=self.DEVICE)
pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32)
cfg = TopKConfig(top_k=k + 1, num_fused_shared_experts=1)
orig_skip = topk_mod._skip_hip_pad_mask
orig_aiter = topk_mod._use_aiter
topk_mod._skip_hip_pad_mask = True
topk_mod._use_aiter = True
try:
out_ids, out_weights, _ = _post_process_topk_ids(
topk_ids,
topk_weights,
cfg,
router_logits,
layer_id=0,
num_token_non_padded=pad,
)
finally:
topk_mod._skip_hip_pad_mask = orig_skip
topk_mod._use_aiter = orig_aiter
self.assertTrue(torch.all(out_ids[n_valid:, :k] == 0))
self.assertTrue(torch.all(out_weights[n_valid:] > 0))
@unittest.skipUnless(torch.cuda.is_available(), "padded-region masking needs a GPU")
@unittest.skipIf(_IS_HIP, "HIP keeps the pre-mask routing contract (AITER/MORI)")
@@ -0,0 +1,46 @@
import unittest
from types import SimpleNamespace
import torch
from sglang.srt.layers.quantization.unquant import _prefer_triton_a16w16
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _tensor(rows, cols, *, dtype=torch.bfloat16, device="cuda:0", contiguous=True):
return SimpleNamespace(
shape=(rows, cols),
dtype=dtype,
device=device,
is_cuda=device.startswith("cuda"),
dim=lambda: 2,
is_contiguous=lambda: contiguous,
)
class TestAiterA16W16Gate(CustomTestCase):
def test_supported_decode_shapes(self):
for rows, k in ((1, 2048), (64, 2048), (512, 512)):
with self.subTest(rows=rows, k=k):
self.assertTrue(_prefer_triton_a16w16(_tensor(rows, k), _tensor(16, k)))
def test_unsupported_shapes(self):
cases = (
(_tensor(65, 2048), _tensor(16, 2048)),
(_tensor(513, 512), _tensor(16, 512)),
(_tensor(1, 2049), _tensor(16, 2049)),
(_tensor(1, 2048), _tensor(16, 1024)),
(_tensor(0, 2048), _tensor(16, 2048)),
(_tensor(1, 2048, device="cpu"), _tensor(16, 2048, device="cpu")),
(_tensor(1, 2048, contiguous=False), _tensor(16, 2048)),
)
for x, weight in cases:
with self.subTest(x=x.shape, weight=weight.shape):
self.assertFalse(_prefer_triton_a16w16(x, weight))
if __name__ == "__main__":
unittest.main()
@@ -23,6 +23,7 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
_CAN_FUSE = forward_mla_rocm._can_fuse_bmm_rope_cat_and_cache
_ABSORB_CONFIG = forward_mla_rocm._absorb_bmm_config
def _attn(**overrides):
@@ -90,5 +91,39 @@ class TestFusedAbsorbGate(CustomTestCase):
self.assertFalse(_CAN_FUSE(_attn(**{name: broken})))
class TestAbsorbBmmConfig(CustomTestCase):
def setUp(self):
self._saved = forward_mla_rocm._absorb_bmm_num_cu
forward_mla_rocm._absorb_bmm_num_cu = 256
self.addCleanup(
setattr,
forward_mla_rocm,
"_absorb_bmm_num_cu",
self._saved,
)
def test_decode_tiles(self):
cases = (
((16, 1, 512), (16, 32, 1)),
((16, 48, 512), (16, 128, 1)),
((16, 200, 512), (32, 128, 2)),
)
for args, expected in cases:
with self.subTest(args=args):
config = _ABSORB_CONFIG(*args)
self.assertEqual(
(
config["BLOCK_SIZE_M"],
config["BLOCK_SIZE_N"],
config["waves_per_eu"],
),
expected,
)
def test_large_row_count_falls_back(self):
self.assertIsNone(_ABSORB_CONFIG(16, 0, 512))
self.assertIsNone(_ABSORB_CONFIG(16, 257, 512))
if __name__ == "__main__":
unittest.main()