[ROCm] GLM-5.2 decode path: decode-shaped MoE/MLA tiles, split speculative softmax, and bf16 GEMM routing (#41725)
Co-authored-by: kyle-256 <Kyle.Zhao@amd.com>
This commit is contained in:
co-authored by
kyle-256
parent
7a719e9a65
commit
aa0ce20201
@@ -1288,10 +1288,12 @@ def _fused_append_shared_experts_kernel(
|
||||
out_weights_ptr,
|
||||
N_BASE, # runtime scalar
|
||||
scale_factor, # runtime scalar
|
||||
num_token_non_padded_ptr, # 1-elem int tensor; only read when HAS_PADDING
|
||||
K: tl.constexpr,
|
||||
S: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_S: tl.constexpr,
|
||||
HAS_PADDING: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
for m in range(M):
|
||||
@@ -1317,22 +1319,35 @@ def _fused_append_shared_experts_kernel(
|
||||
ids = tl.load(topk_ids_ptr + ids_row_ptr + offs_k, mask=mask_k)
|
||||
ws = tl.load(topk_weights_ptr + w_row_ptr + offs_k, mask=mask_k)
|
||||
|
||||
tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k)
|
||||
tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k)
|
||||
|
||||
offs_s = tl.arange(0, BLOCK_S)
|
||||
mask_s = offs_s < S
|
||||
|
||||
shared_ids = tl.cast(N_BASE + offs_s, ids.dtype)
|
||||
shared_ws = tl.full([BLOCK_S], scale_factor, dtype=ws.dtype)
|
||||
|
||||
if HAS_PADDING:
|
||||
# Padded rows zero routed ids and all weights.
|
||||
if pid >= tl.load(num_token_non_padded_ptr):
|
||||
ids = tl.zeros([BLOCK_K], dtype=ids.dtype)
|
||||
ws = tl.zeros([BLOCK_K], dtype=ws.dtype)
|
||||
shared_ws = tl.zeros([BLOCK_S], dtype=ws.dtype)
|
||||
|
||||
tl.store(out_ids_ptr + out_ids_row_ptr + offs_k, ids, mask=mask_k)
|
||||
tl.store(out_weights_ptr + out_w_row_ptr + offs_k, ws, mask=mask_k)
|
||||
|
||||
tl.store(out_ids_ptr + out_ids_row_ptr + K + offs_s, shared_ids, mask=mask_s)
|
||||
tl.store(out_weights_ptr + out_w_row_ptr + K + offs_s, shared_ws, mask=mask_s)
|
||||
|
||||
|
||||
def fused_append_shared_experts(
|
||||
topk_ids, topk_weights, num_fused_shared_experts, scale_factor, N=None
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_fused_shared_experts,
|
||||
scale_factor,
|
||||
N=None,
|
||||
num_token_non_padded=None,
|
||||
):
|
||||
"""Append shared experts and optionally materialize padded rows."""
|
||||
assert N is not None, "N (shared expert base id) must be provided"
|
||||
m, k = topk_ids.shape
|
||||
s = int(num_fused_shared_experts)
|
||||
@@ -1344,6 +1359,8 @@ def fused_append_shared_experts(
|
||||
(m, k + s), dtype=topk_weights.dtype, device=topk_weights.device
|
||||
)
|
||||
|
||||
has_padding = num_token_non_padded is not None
|
||||
ntnp_ptr = num_token_non_padded if has_padding else topk_ids
|
||||
_fused_append_shared_experts_kernel[(m,)](
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
@@ -1351,10 +1368,12 @@ def fused_append_shared_experts(
|
||||
out_weights,
|
||||
N_BASE=N,
|
||||
scale_factor=scale_factor,
|
||||
num_token_non_padded_ptr=ntnp_ptr,
|
||||
K=k,
|
||||
S=s,
|
||||
BLOCK_K=triton.next_power_of_2(k),
|
||||
BLOCK_S=triton.next_power_of_2(s),
|
||||
HAS_PADDING=has_padding,
|
||||
num_warps=1,
|
||||
)
|
||||
return out_ids, out_weights
|
||||
|
||||
@@ -40,10 +40,21 @@ def cake_softmax(logits: torch.Tensor) -> torch.Tensor:
|
||||
return get_kernel("sampling.softmax", KernelBackend.FLASHINFER)(logits)
|
||||
|
||||
|
||||
def softmax(logits: torch.Tensor) -> torch.Tensor:
|
||||
"""Softmax for sampler logits, with a qualified Blackwell fast path."""
|
||||
def softmax(
|
||||
logits: torch.Tensor, temperatures: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Softmax for sampler logits, optionally with per-row temperatures."""
|
||||
import torch
|
||||
|
||||
if temperatures is not None and logits.is_cuda and torch.version.hip is not None:
|
||||
from sglang.kernels.ops.sampling.temperature_softmax import (
|
||||
temperature_softmax,
|
||||
)
|
||||
|
||||
return temperature_softmax(logits, temperatures)
|
||||
|
||||
if temperatures is not None:
|
||||
logits = logits / temperatures
|
||||
from sglang.kernels.cake_kernels.sampling import supports_softmax
|
||||
|
||||
if supports_softmax(logits):
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Split-row temperature softmax for low-row-count sampling workloads."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
# Split low-row-count reductions across CTAs and fuse temperature scaling.
|
||||
_MIN_BLOCK = 2048
|
||||
_TARGET_CTAS = 512
|
||||
ROW_LIMIT = 32
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _partial_max_sumexp_kernel(
|
||||
logits,
|
||||
temperatures,
|
||||
partial_max,
|
||||
partial_sum,
|
||||
logits_row_stride,
|
||||
vocab_size,
|
||||
num_splits,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
split = tl.program_id(1)
|
||||
t = tl.load(temperatures + row)
|
||||
offsets = split * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offsets < vocab_size
|
||||
x = tl.load(
|
||||
logits + row * logits_row_stride + offsets,
|
||||
mask=mask,
|
||||
other=-float("inf"),
|
||||
)
|
||||
x = x / t
|
||||
block_max = tl.max(x, axis=0)
|
||||
# Fully masked splits contribute zero without evaluating -inf - (-inf).
|
||||
safe_block_max = tl.where(block_max == -float("inf"), 0.0, block_max)
|
||||
block_sum = tl.sum(tl.where(mask, tl.exp(x - safe_block_max), 0.0), axis=0)
|
||||
out_offset = row * num_splits + split
|
||||
tl.store(partial_max + out_offset, block_max)
|
||||
tl.store(partial_sum + out_offset, block_sum)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combine_kernel(
|
||||
partial_max,
|
||||
partial_sum,
|
||||
row_max,
|
||||
row_sum,
|
||||
num_splits,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
mask = offsets < num_splits
|
||||
m = tl.load(
|
||||
partial_max + row * num_splits + offsets, mask=mask, other=-float("inf")
|
||||
)
|
||||
s = tl.load(partial_sum + row * num_splits + offsets, mask=mask, other=0.0)
|
||||
m_all = tl.max(m, axis=0)
|
||||
s_all = tl.sum(tl.where(mask, s * tl.exp(m - m_all), 0.0), axis=0)
|
||||
tl.store(row_max + row, m_all)
|
||||
tl.store(row_sum + row, s_all)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _normalize_kernel(
|
||||
logits,
|
||||
temperatures,
|
||||
row_max,
|
||||
row_sum,
|
||||
probs,
|
||||
logits_row_stride,
|
||||
probs_row_stride,
|
||||
vocab_size,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
split = tl.program_id(1)
|
||||
t = tl.load(temperatures + row)
|
||||
m = tl.load(row_max + row)
|
||||
s = tl.load(row_sum + row)
|
||||
offsets = split * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offsets < vocab_size
|
||||
x = tl.load(logits + row * logits_row_stride + offsets, mask=mask, other=0.0)
|
||||
p = tl.exp(x / t - m) / s
|
||||
tl.store(probs + row * probs_row_stride + offsets, p, mask=mask)
|
||||
|
||||
|
||||
def _split_geometry(rows: int, vocab_size: int):
|
||||
"""Choose a block width and split count near the target grid size."""
|
||||
splits = min(
|
||||
triton.cdiv(_TARGET_CTAS, rows), max(1, triton.cdiv(vocab_size, _MIN_BLOCK))
|
||||
)
|
||||
block = max(_MIN_BLOCK, triton.next_power_of_2(triton.cdiv(vocab_size, splits)))
|
||||
return block, triton.cdiv(vocab_size, block)
|
||||
|
||||
|
||||
def temperature_softmax(
|
||||
logits: torch.Tensor, temperatures: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Apply temperature softmax, splitting eligible rows across CTAs."""
|
||||
if (
|
||||
logits.ndim != 2
|
||||
or logits.dtype != torch.float32
|
||||
or logits.shape[0] == 0
|
||||
or logits.shape[1] == 0
|
||||
or logits.shape[0] > ROW_LIMIT
|
||||
or logits.stride(1) != 1
|
||||
or not logits.is_cuda
|
||||
):
|
||||
return torch.softmax(logits / temperatures, dim=-1)
|
||||
|
||||
rows, vocab_size = logits.shape
|
||||
t = temperatures.reshape(-1)
|
||||
if (
|
||||
t.shape[0] != rows
|
||||
or t.stride(0) != 1
|
||||
or t.dtype != logits.dtype
|
||||
or t.device != logits.device
|
||||
):
|
||||
return torch.softmax(logits / temperatures, dim=-1)
|
||||
|
||||
block, num_splits = _split_geometry(rows, vocab_size)
|
||||
if num_splits == 1:
|
||||
return torch.softmax(logits / temperatures, dim=-1)
|
||||
|
||||
probs = torch.empty_like(logits)
|
||||
partial_max = torch.empty(
|
||||
(rows, num_splits), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
partial_sum = torch.empty(
|
||||
(rows, num_splits), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
row_max = torch.empty((rows,), dtype=torch.float32, device=logits.device)
|
||||
row_sum = torch.empty((rows,), dtype=torch.float32, device=logits.device)
|
||||
|
||||
_partial_max_sumexp_kernel[(rows, num_splits)](
|
||||
logits,
|
||||
t,
|
||||
partial_max,
|
||||
partial_sum,
|
||||
logits.stride(0),
|
||||
vocab_size,
|
||||
num_splits,
|
||||
BLOCK=block,
|
||||
num_warps=8,
|
||||
)
|
||||
_combine_kernel[(rows,)](
|
||||
partial_max,
|
||||
partial_sum,
|
||||
row_max,
|
||||
row_sum,
|
||||
num_splits,
|
||||
BLOCK=triton.next_power_of_2(num_splits),
|
||||
num_warps=1,
|
||||
)
|
||||
_normalize_kernel[(rows, num_splits)](
|
||||
logits,
|
||||
t,
|
||||
row_max,
|
||||
row_sum,
|
||||
probs,
|
||||
logits.stride(0),
|
||||
probs.stride(0),
|
||||
vocab_size,
|
||||
BLOCK=block,
|
||||
num_warps=8,
|
||||
)
|
||||
return probs
|
||||
@@ -2291,8 +2291,14 @@ def _post_process_topk_ids(
|
||||
)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, routed_topk_ids)
|
||||
recorder_topk_ids = None
|
||||
_aiter_append = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and topk_ids.shape[-1] < topk_config.top_k
|
||||
)
|
||||
_fold_pad_into_append = False
|
||||
recorder_was_fused = False
|
||||
_fold_pad_weights_into_append = False
|
||||
if _is_cuda:
|
||||
# LP path: solve LP outside torch.compile (the solver contains an
|
||||
# EP all-reduce that can't run inside compiled regions).
|
||||
@@ -2344,21 +2350,20 @@ def _post_process_topk_ids(
|
||||
# Regression: skipping this mask when EPLB is disabled caused garbage
|
||||
# MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+).
|
||||
#
|
||||
# Fold: when the fused append+remap kernel runs below (aiter per-rank
|
||||
# shared-slot path, EPLB off) it folds this padded fill itself
|
||||
# (pad_fill_id=0 -> remap(0)=0, bit-identical), so skip the separate
|
||||
# _fill_padded_rows launch here.
|
||||
# Let append kernels materialize padded ids when remapping is disabled.
|
||||
eplb_remap_enabled = _eplb_remap_enabled()
|
||||
_fold_pad_into_append = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and use_per_rank_shared_slots
|
||||
and not _eplb_remap_enabled()
|
||||
_aiter_append
|
||||
and not eplb_remap_enabled
|
||||
and (use_per_rank_shared_slots or not _skip_hip_pad_mask)
|
||||
)
|
||||
_fold_pad_weights_into_append = (
|
||||
_fold_pad_into_append and not use_per_rank_shared_slots
|
||||
)
|
||||
if not _fold_pad_into_append:
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0)
|
||||
if (
|
||||
_is_hip
|
||||
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
|
||||
envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
|
||||
and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get()
|
||||
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
|
||||
and layer_id is not None
|
||||
@@ -2383,7 +2388,7 @@ def _post_process_topk_ids(
|
||||
# The logical->physical remap is only meaningful when a real
|
||||
# expert-location mapping exists. With a trivial placement and EPLB off
|
||||
# the map is identity so the remap can be skipped safely.
|
||||
if not recorder_was_fused and _eplb_remap_enabled():
|
||||
if not recorder_was_fused and eplb_remap_enabled:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info
|
||||
)
|
||||
@@ -2397,14 +2402,6 @@ def _post_process_topk_ids(
|
||||
topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids
|
||||
)
|
||||
|
||||
# The JIT grouped router and the ROCm decode gate emit the shared slots themselves;
|
||||
# appending again would write the shared id twice and evict a real routed expert.
|
||||
_aiter_append = (
|
||||
num_fused_shared_experts > 0
|
||||
and _use_aiter
|
||||
and topk_ids.shape[-1] < topk_config.top_k
|
||||
)
|
||||
|
||||
if _aiter_append and use_per_rank_shared_slots:
|
||||
# Fused path: append shared experts AND apply the per-rank shared-slot
|
||||
# remap in a single Triton kernel. This replaces the original
|
||||
@@ -2447,7 +2444,7 @@ def _post_process_topk_ids(
|
||||
),
|
||||
)
|
||||
elif _aiter_append:
|
||||
M, N = router_logits.shape
|
||||
N = router_logits.shape[1]
|
||||
scale_factor = (
|
||||
1.0
|
||||
if fused_shared_experts_scaling_factor is None
|
||||
@@ -2465,6 +2462,9 @@ def _post_process_topk_ids(
|
||||
num_fused_shared_experts,
|
||||
scale_factor,
|
||||
N, # base id for shared experts
|
||||
num_token_non_padded=(
|
||||
num_token_non_padded if _fold_pad_weights_into_append else None
|
||||
),
|
||||
)
|
||||
|
||||
elif use_per_rank_shared_slots:
|
||||
@@ -2491,7 +2491,7 @@ def _post_process_topk_ids(
|
||||
fused_shared_experts_scaling_factor
|
||||
)
|
||||
|
||||
if _is_hip and not _skip_hip_pad_mask:
|
||||
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
|
||||
# Shared-expert append/remap can introduce non-zero weights after the
|
||||
# initial HIP padding mask above. Ensure padded tokens leave this helper
|
||||
# with all expert weights zeroed.
|
||||
|
||||
@@ -68,11 +68,44 @@ _is_hip = is_hip()
|
||||
_is_cpu = is_cpu()
|
||||
_is_npu = is_npu()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
_is_gfx95 = _is_hip and is_gfx95_supported()
|
||||
|
||||
if _use_aiter:
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16 as _aiter_triton_a16w16
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
# Route supported decode shapes that are absent from the tuned table.
|
||||
_A16W16_TRITON_MAX_K = 2048
|
||||
_A16W16_TRITON_MAX_M = 64
|
||||
_A16W16_TRITON_NARROW_K = 512
|
||||
_A16W16_TRITON_NARROW_K_MAX_M = 512
|
||||
|
||||
|
||||
def _prefer_triton_a16w16(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
"""Return whether the Triton BF16 path supports this decode shape."""
|
||||
if x.dim() != 2 or weight.dim() != 2:
|
||||
return False
|
||||
k = weight.shape[1]
|
||||
max_m = (
|
||||
_A16W16_TRITON_NARROW_K_MAX_M
|
||||
if k <= _A16W16_TRITON_NARROW_K
|
||||
else _A16W16_TRITON_MAX_M
|
||||
)
|
||||
return (
|
||||
x.is_cuda
|
||||
and weight.is_cuda
|
||||
and x.device == weight.device
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and x.shape[1] == k
|
||||
and 0 < k <= _A16W16_TRITON_MAX_K
|
||||
and 0 < x.shape[0] <= max_m
|
||||
and weight.shape[0] > 0
|
||||
and x.is_contiguous()
|
||||
and weight.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
class Bf16GemmBackend(Enum):
|
||||
AUTO = "auto"
|
||||
@@ -479,6 +512,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
return output
|
||||
|
||||
elif _use_aiter and type(layer.weight.data) is torch.Tensor:
|
||||
if _is_gfx95 and _prefer_triton_a16w16(x, layer.weight):
|
||||
return _aiter_triton_a16w16(x, layer.weight, bias, dtype=x.dtype)
|
||||
return tgemm.mm(x, layer.weight, bias, otype=x.dtype)
|
||||
|
||||
elif (
|
||||
|
||||
+45
-1
@@ -66,7 +66,12 @@ from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.state_capturer.indexer_topk import (
|
||||
maybe_capture_indexer_topk,
|
||||
)
|
||||
from sglang.srt.utils import BumpAllocator, get_bool_env_var
|
||||
from sglang.srt.utils import (
|
||||
BumpAllocator,
|
||||
cdiv,
|
||||
get_bool_env_var,
|
||||
get_device_core_count,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
|
||||
@@ -145,6 +150,39 @@ if _use_aiter_gfx95:
|
||||
)
|
||||
|
||||
|
||||
_ABSORB_BMM_DECODE_MAX_M = 256
|
||||
_absorb_bmm_num_cu = 0
|
||||
|
||||
|
||||
def _absorb_bmm_config(heads: int, m: int, n: int) -> Optional[dict]:
|
||||
"""Return decode tile settings, or None outside the decode row limit."""
|
||||
global _absorb_bmm_num_cu
|
||||
if m <= 0 or m > _ABSORB_BMM_DECODE_MAX_M:
|
||||
return None
|
||||
if not _absorb_bmm_num_cu:
|
||||
_absorb_bmm_num_cu = get_device_core_count()
|
||||
num_cu = _absorb_bmm_num_cu
|
||||
m_tiles_16 = cdiv(m, 16)
|
||||
block_n = 128
|
||||
while block_n > 32 and 4 * heads * m_tiles_16 * cdiv(n, block_n) < 3 * num_cu:
|
||||
block_n //= 2
|
||||
n_tiles = cdiv(n, block_n)
|
||||
block_m = 16
|
||||
while block_m < 64 and heads * cdiv(m, block_m) * n_tiles > 2 * num_cu:
|
||||
block_m *= 2
|
||||
workgroups = heads * cdiv(m, block_m) * n_tiles
|
||||
return {
|
||||
"BLOCK_SIZE_M": block_m,
|
||||
"BLOCK_SIZE_N": block_n,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": max(1, cdiv(workgroups, num_cu)),
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
}
|
||||
|
||||
|
||||
def _absorb_weight_bf16(w: torch.Tensor, w_scale) -> torch.Tensor:
|
||||
"""Dequantize an absorbed MLA weight, skipping the pass when it is a no-op."""
|
||||
if (
|
||||
@@ -196,6 +234,9 @@ def rocm_absorb_q_bmm(
|
||||
transpose_bm=False, # (B, M, N)
|
||||
transpose_bm_in=True, # (M, B, K)
|
||||
dtype=torch.bfloat16,
|
||||
config=_absorb_bmm_config(
|
||||
q_nope.shape[1], q_nope.shape[0], attn.w_kc.shape[2]
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -254,6 +295,9 @@ def rocm_absorb_v_bmm(
|
||||
transpose_bm=True,
|
||||
transpose_bm_in=True,
|
||||
dtype=torch.bfloat16,
|
||||
config=_absorb_bmm_config(
|
||||
attn.num_local_heads, attn_output.shape[0], attn.w_vc.shape[-1]
|
||||
),
|
||||
)
|
||||
elif not is_in_tc_piecewise_cuda_graph():
|
||||
# Same (batch, heads, dim) layout as the quantized paths above, so the
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax as sampling_softmax
|
||||
from sglang.kernels.ops.speculative.spec_tree import (
|
||||
sgl_build_tree_kernel_efficient_triton,
|
||||
verify_tree_greedy_kernel_triton,
|
||||
@@ -743,8 +744,6 @@ def eagle_sample(
|
||||
Verify and find accepted tokens based on logits output and batch
|
||||
(which contains spec decoding information).
|
||||
"""
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -927,8 +926,8 @@ def eagle_sample(
|
||||
sampling_info.temperatures, verify_input.draft_token_num, dim=0
|
||||
) # (bs * num_draft_tokens, 1)
|
||||
|
||||
target_probs = F.softmax(
|
||||
next_token_logits / expanded_temperature, dim=-1
|
||||
target_probs = sampling_softmax(
|
||||
next_token_logits, temperatures=expanded_temperature
|
||||
) # (bs * num_draft_tokens, vocab_size)
|
||||
maybe_detect_nan(target_probs, "v2 verify: target_probs after softmax")
|
||||
if sampling_info.need_top_k_sampling:
|
||||
@@ -1054,8 +1053,6 @@ def eagle_sample(
|
||||
def eagle_prepare_for_decode(batch: ScheduleBatch):
|
||||
batch.maybe_evict_swa()
|
||||
|
||||
bs = batch.batch_size()
|
||||
|
||||
# Accumulate penalty
|
||||
# This is a relaxed version of penalties for speculative decoding.
|
||||
if batch.sampling_info.penalizer_orchestrator.is_required:
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Tuple
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax as sampling_softmax
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
align_evict_mask_to_page_size as align_evict_mask_to_page_size,
|
||||
)
|
||||
@@ -31,6 +32,7 @@ from sglang.kernels.ops.speculative.cache_locs import (
|
||||
from sglang.kernels.ops.speculative.eagle import (
|
||||
fill_accept_out_cache_loc_func as fill_accept_out_cache_loc_func,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.row_argmax import row_argmax
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.constrained.base_grammar_backend import GrammarMask
|
||||
@@ -138,7 +140,7 @@ def fast_sample(probs: torch.Tensor, num_samples: int = 1):
|
||||
q.clamp_min_(torch.finfo(torch.float32).tiny)
|
||||
scores = probs.float() / q
|
||||
if num_samples == 1:
|
||||
sample_index = scores.argmax(dim=-1, keepdim=True)
|
||||
sample_index = row_argmax(scores).unsqueeze(-1)
|
||||
else:
|
||||
sample_index = scores.topk(num_samples, dim=-1).indices
|
||||
sample_p = probs.gather(1, sample_index)
|
||||
@@ -157,8 +159,8 @@ def renorm_draft_probs(
|
||||
temperature (higher acceptance; correctness holds for any q).
|
||||
"""
|
||||
if not use_rejection_sampling or not next_token_logits.size(0):
|
||||
return torch.softmax(next_token_logits, dim=-1)
|
||||
return torch.softmax(next_token_logits / sampling_info.temperatures, dim=-1)
|
||||
return sampling_softmax(next_token_logits)
|
||||
return sampling_softmax(next_token_logits, temperatures=sampling_info.temperatures)
|
||||
|
||||
|
||||
def sample_draft_proposal(
|
||||
@@ -186,7 +188,7 @@ def sample_draft_proposal(
|
||||
(p - q)+ it resamples from is p itself. Both arms commit the target argmax,
|
||||
which is what greedy means. Drop that renorm and this stops holding.
|
||||
"""
|
||||
probs = torch.softmax(next_token_logits / temperatures, dim=-1)
|
||||
probs = sampling_softmax(next_token_logits, temperatures=temperatures)
|
||||
topk_p, topk_index = fast_sample(probs, num_samples=1)
|
||||
if top_ks is not None:
|
||||
# Assert rather than skip on a device mismatch: a host-side top_ks would
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=10, stage="stage-b", runner_config="1-gpu-small-amd")
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required for this test.")
|
||||
class TestTemperatureSoftmax(CustomTestCase):
|
||||
@staticmethod
|
||||
def temperatures(rows: int) -> torch.Tensor:
|
||||
return torch.linspace(
|
||||
0.5, 1.5, rows, dtype=torch.float32, device="cuda"
|
||||
).unsqueeze(1)
|
||||
|
||||
def assert_matches_torch(self, logits: torch.Tensor, temperatures: torch.Tensor):
|
||||
expected = torch.softmax(logits / temperatures, dim=-1)
|
||||
actual = softmax(logits, temperatures=temperatures)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=1e-7)
|
||||
|
||||
def test_dense_logits(self):
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
for rows, vocab_size in (
|
||||
(0, 1024),
|
||||
(1, 0),
|
||||
(1, 1024),
|
||||
(1, 32000),
|
||||
(7, 65537),
|
||||
(32, 128257),
|
||||
(33, 4096),
|
||||
):
|
||||
with self.subTest(rows=rows, vocab_size=vocab_size):
|
||||
logits = torch.randn(
|
||||
(rows, vocab_size),
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
generator=generator,
|
||||
)
|
||||
self.assert_matches_torch(logits, self.temperatures(rows))
|
||||
|
||||
def test_noncontiguous_logits_fall_back(self):
|
||||
logits = torch.randn((4096, 3), dtype=torch.float32, device="cuda").T
|
||||
self.assert_matches_torch(logits, self.temperatures(logits.shape[0]))
|
||||
|
||||
def test_sparse_grammar_mask_with_fully_masked_splits(self):
|
||||
vocab_size = 65537
|
||||
allowed = torch.tensor(
|
||||
[0, vocab_size // 2, vocab_size - 1], device="cuda", dtype=torch.long
|
||||
)
|
||||
|
||||
for rows in (1, 3, 16, 32):
|
||||
with self.subTest(rows=rows):
|
||||
logits = torch.full(
|
||||
(rows, vocab_size),
|
||||
-float("inf"),
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
)
|
||||
values = torch.arange(
|
||||
rows * allowed.numel(), dtype=torch.float32, device="cuda"
|
||||
).reshape(rows, allowed.numel())
|
||||
values = values.remainder(7).sub_(3)
|
||||
logits[:, allowed] = values
|
||||
self.assert_matches_torch(logits, self.temperatures(rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -101,6 +101,31 @@ class TestFusedAppendSharedExpertsTop6(CustomTestCase):
|
||||
self.assertTrue(torch.equal(got_ids[:, :k], topk_ids))
|
||||
self.assertTrue(torch.allclose(got_w[:, :k], topk_weights))
|
||||
|
||||
def test_padded_rows(self):
|
||||
m, k, s = 8, 6, 2
|
||||
topk_ids, topk_weights = self._make_inputs(m, k)
|
||||
for valid_rows in (0, 3, m):
|
||||
with self.subTest(valid_rows=valid_rows):
|
||||
num_token_non_padded = torch.tensor(
|
||||
[valid_rows], dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
got_ids, got_w = fused_append_shared_experts(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
s,
|
||||
0.5,
|
||||
N=self.N_BASE,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
)
|
||||
exp_ids, exp_w = _reference_append(
|
||||
topk_ids, topk_weights, s, 0.5, self.N_BASE
|
||||
)
|
||||
exp_ids[valid_rows:, :k] = 0
|
||||
exp_w[valid_rows:] = 0
|
||||
|
||||
self.assertTrue(torch.equal(got_ids, exp_ids))
|
||||
self.assertTrue(torch.equal(got_w, exp_w))
|
||||
|
||||
def test_no_shared_experts_is_noop(self):
|
||||
"""s == 0 returns the inputs untouched (no kernel launch)."""
|
||||
topk_ids, topk_weights = self._make_inputs(4, 6)
|
||||
|
||||
@@ -189,6 +189,33 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase):
|
||||
finally:
|
||||
topk_mod._skip_hip_pad_mask = orig
|
||||
|
||||
def test_flag_skips_weight_masking_with_shared_append(self):
|
||||
n, k, n_valid = 16, 8, 5
|
||||
topk_weights = torch.rand((n, k), device=self.DEVICE) + 0.5
|
||||
topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32)
|
||||
router_logits = torch.rand((n, 64), device=self.DEVICE)
|
||||
pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32)
|
||||
cfg = TopKConfig(top_k=k + 1, num_fused_shared_experts=1)
|
||||
orig_skip = topk_mod._skip_hip_pad_mask
|
||||
orig_aiter = topk_mod._use_aiter
|
||||
topk_mod._skip_hip_pad_mask = True
|
||||
topk_mod._use_aiter = True
|
||||
try:
|
||||
out_ids, out_weights, _ = _post_process_topk_ids(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
cfg,
|
||||
router_logits,
|
||||
layer_id=0,
|
||||
num_token_non_padded=pad,
|
||||
)
|
||||
finally:
|
||||
topk_mod._skip_hip_pad_mask = orig_skip
|
||||
topk_mod._use_aiter = orig_aiter
|
||||
|
||||
self.assertTrue(torch.all(out_ids[n_valid:, :k] == 0))
|
||||
self.assertTrue(torch.all(out_weights[n_valid:] > 0))
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "padded-region masking needs a GPU")
|
||||
@unittest.skipIf(_IS_HIP, "HIP keeps the pre-mask routing contract (AITER/MORI)")
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization.unquant import _prefer_triton_a16w16
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _tensor(rows, cols, *, dtype=torch.bfloat16, device="cuda:0", contiguous=True):
|
||||
return SimpleNamespace(
|
||||
shape=(rows, cols),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
is_cuda=device.startswith("cuda"),
|
||||
dim=lambda: 2,
|
||||
is_contiguous=lambda: contiguous,
|
||||
)
|
||||
|
||||
|
||||
class TestAiterA16W16Gate(CustomTestCase):
|
||||
def test_supported_decode_shapes(self):
|
||||
for rows, k in ((1, 2048), (64, 2048), (512, 512)):
|
||||
with self.subTest(rows=rows, k=k):
|
||||
self.assertTrue(_prefer_triton_a16w16(_tensor(rows, k), _tensor(16, k)))
|
||||
|
||||
def test_unsupported_shapes(self):
|
||||
cases = (
|
||||
(_tensor(65, 2048), _tensor(16, 2048)),
|
||||
(_tensor(513, 512), _tensor(16, 512)),
|
||||
(_tensor(1, 2049), _tensor(16, 2049)),
|
||||
(_tensor(1, 2048), _tensor(16, 1024)),
|
||||
(_tensor(0, 2048), _tensor(16, 2048)),
|
||||
(_tensor(1, 2048, device="cpu"), _tensor(16, 2048, device="cpu")),
|
||||
(_tensor(1, 2048, contiguous=False), _tensor(16, 2048)),
|
||||
)
|
||||
for x, weight in cases:
|
||||
with self.subTest(x=x.shape, weight=weight.shape):
|
||||
self.assertFalse(_prefer_triton_a16w16(x, weight))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -23,6 +23,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
_CAN_FUSE = forward_mla_rocm._can_fuse_bmm_rope_cat_and_cache
|
||||
_ABSORB_CONFIG = forward_mla_rocm._absorb_bmm_config
|
||||
|
||||
|
||||
def _attn(**overrides):
|
||||
@@ -90,5 +91,39 @@ class TestFusedAbsorbGate(CustomTestCase):
|
||||
self.assertFalse(_CAN_FUSE(_attn(**{name: broken})))
|
||||
|
||||
|
||||
class TestAbsorbBmmConfig(CustomTestCase):
|
||||
def setUp(self):
|
||||
self._saved = forward_mla_rocm._absorb_bmm_num_cu
|
||||
forward_mla_rocm._absorb_bmm_num_cu = 256
|
||||
self.addCleanup(
|
||||
setattr,
|
||||
forward_mla_rocm,
|
||||
"_absorb_bmm_num_cu",
|
||||
self._saved,
|
||||
)
|
||||
|
||||
def test_decode_tiles(self):
|
||||
cases = (
|
||||
((16, 1, 512), (16, 32, 1)),
|
||||
((16, 48, 512), (16, 128, 1)),
|
||||
((16, 200, 512), (32, 128, 2)),
|
||||
)
|
||||
for args, expected in cases:
|
||||
with self.subTest(args=args):
|
||||
config = _ABSORB_CONFIG(*args)
|
||||
self.assertEqual(
|
||||
(
|
||||
config["BLOCK_SIZE_M"],
|
||||
config["BLOCK_SIZE_N"],
|
||||
config["waves_per_eu"],
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
def test_large_row_count_falls_back(self):
|
||||
self.assertIsNone(_ABSORB_CONFIG(16, 0, 512))
|
||||
self.assertIsNone(_ABSORB_CONFIG(16, 257, 512))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user