[Perf] Optimize DeepSeek V4.1 Flash Hopper paths and Blackwell prefill selection (#41251)

(cherry picked from commit 391e665dab)
This commit is contained in:
Xiaoyu Zhang
2026-10-19 10:13:35 +08:00
committed by minke.yu
parent 9247c97919
commit 2b192aade6
31 changed files with 2093 additions and 183 deletions
@@ -53,6 +53,7 @@ struct ActivationParams {
// for per-token routing and BLOCK_SIZE_M for sorted/TMA routing.
const int32_t* __restrict__ expert_ids;
uint32_t expert_step;
float clamp_limit;
};
template <
@@ -61,7 +62,8 @@ template <
bool kUsePDL,
bool kFilterExpert,
bool kRoundActivation = false,
bool kReuseInput = false>
bool kReuseInput = false,
bool kClamp = false>
__global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams params) {
using namespace device;
constexpr auto kVecSize = kMaxVecBytes / sizeof(T);
@@ -80,11 +82,17 @@ __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams para
PDLWaitPrimary<kUsePDL>();
const auto gate = device::load_as<vec_t>(params.input, input_offset);
const auto up = device::load_as<vec_t>(params.input, input_offset + num_vecs);
const float limit = device::cast<fp32_t>(device::cast<T>(params.clamp_limit));
vec_t out;
#pragma unroll
for (int i = 0; i < kVecSize; ++i) {
const float gate_f32 = device::cast<fp32_t>(gate[i]);
const float up_f32 = device::cast<fp32_t>(up[i]);
float gate_f32 = device::cast<fp32_t>(gate[i]);
float up_f32 = device::cast<fp32_t>(up[i]);
if constexpr (kClamp) {
static_assert(kAct == ActivationKind::kSiLU);
gate_f32 = gate_f32 > limit ? limit : gate_f32;
up_f32 = up_f32 > limit ? limit : (up_f32 < -limit ? -limit : up_f32);
}
if constexpr (kRoundActivation) {
const T activated = device::cast<T>(apply_activation_f32<kAct>(gate_f32));
out[i] = device::cast<T>(device::cast<fp32_t>(activated) * up_f32);
@@ -153,13 +161,14 @@ struct ActivationKernel {
return nullptr;
}
template <bool kRoundActivation = false, bool kReuseInput = false>
template <bool kRoundActivation = false, bool kReuseInput = false, bool kClamp = false>
static void launch(
const tvm::ffi::TensorView& input,
const tvm::ffi::TensorView& out,
const std::string& type,
const int32_t* expert_ids,
uint32_t expert_step) {
uint32_t expert_step,
float clamp_limit = 0.0f) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
@@ -198,8 +207,14 @@ struct ActivationKernel {
.num_tokens = num_tokens,
.expert_ids = expert_ids,
.expert_step = expert_step,
.clamp_limit = clamp_limit,
};
if (expert_ids != nullptr) {
if constexpr (kClamp) {
RuntimeCheck(type == "silu" && expert_ids == nullptr, "clamping requires unfiltered SiLU");
const auto kernel =
act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false, kRoundActivation, kReuseInput, true>;
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
} else if (expert_ids != nullptr) {
RuntimeCheck(expert_step > 0, "expert_step must be positive");
const auto kernel = select_kernel<true, kRoundActivation, kReuseInput>(type);
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
@@ -213,9 +228,13 @@ struct ActivationKernel {
launch(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
}
static void
run_activation_with_rounding(const tvm::ffi::TensorView input, const tvm::ffi::TensorView out, std::string type) {
launch<true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
static void run_activation_with_rounding(
const tvm::ffi::TensorView input, const tvm::ffi::TensorView out, std::string type, double clamp_limit) {
if (clamp_limit > 0) {
launch<true, false, true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1, clamp_limit);
} else {
launch<true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
}
}
static void run_activation_with_rounding_input_inplace(
@@ -404,6 +404,7 @@ bool is_valid_config(
#define MXFP4_GET_IF(W_TYPE) \
MXFP4_GET_IF_M1(W_TYPE, 8, 8, 256) \
MXFP4_GET_IF_M1(W_TYPE, 8, 4, 128) \
MXFP4_GET_IF_M1(W_TYPE, 4, 8, 128) \
MXFP4_GET_IF_M234(W_TYPE, 16, 4, 256) \
MXFP4_GET_IF_M234(W_TYPE, 8, 4, 128)
@@ -492,6 +493,16 @@ exec_config_t determine_exec_config(
bool is_zp_float,
int max_shared_mem,
int sms) {
#if SGL_CUDA_ARCH == 900
if constexpr (std::is_same_v<scalar_t, __nv_bfloat16> && !kIsEP && !kHasBias) {
// H200 TP8 DeepSeek-V4.1 gate/up: six sparse expert rows need a narrower
// N tile and fewer persistent blocks than the occupancy-only heuristic.
if (q_type == host::kFE2M1f && group_size == 32 && prob_m == 1 && prob_n == 640 && prob_k == 5120 && top_k == 6 &&
thread_m_blocks == 1 && m_block_size_8 && !has_act_order && !has_zp) {
return exec_config_t{2, thread_config_t{128, 64, 128}};
}
}
#endif
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs;
int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t)
@@ -67,14 +67,14 @@ def _run_activation_inplace(
@register_custom_op(mutates_args=["out"])
def _run_activation_with_rounding_inplace(
op_name: str, input: torch.Tensor, out: torch.Tensor
op_name: str, input: torch.Tensor, out: torch.Tensor, clamp_limit: float = 0.0
) -> None:
hidden_size = input.shape[-1] // 2
# Fast-math changes FP16 SiLU at eager rounding boundaries on SM90.
module = activation_module(input.dtype, fast_math=False)
input_2d = input.view(-1, hidden_size * 2)
out_2d = out.view(-1, hidden_size)
module.run_activation_with_rounding(input_2d, out_2d, op_name)
module.run_activation_with_rounding(input_2d, out_2d, op_name, clamp_limit)
@register_custom_op(mutates_args=["input"])
@@ -184,11 +184,13 @@ def silu_and_mul(
def silu_and_mul_with_activation_rounding(
input: torch.Tensor,
out: Optional[torch.Tensor] = None,
*,
clamp_limit: float = 0.0,
) -> torch.Tensor:
hidden_size = input.shape[-1] // 2
if out is None:
out = input.new_empty(*input.shape[:-1], hidden_size)
_run_activation_with_rounding_inplace("silu", input, out)
_run_activation_with_rounding_inplace("silu", input, out, clamp_limit)
return out
@@ -359,3 +359,14 @@ register_kernel(
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
register_kernel(
KernelSpec(
op=f"attention.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.attention.dsv4.fp4_indexer:{_fn}",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
del _fn
@@ -143,6 +143,18 @@ def amax_topk_blocks(
return blocks
def candidate_block_mask(
blocks: torch.Tensor, width: int, block_size: int
) -> torch.Tensor:
"""Materialize a source's block IDs once for Hopper paged-score consumers."""
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
)
keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True)
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
def select_candidate_block_ids(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
@@ -15,6 +15,170 @@ INDEX_K_SCALE_BYTES = tl.constexpr(4)
INDEX_K_SLOT_BYTES = INDEX_K_PAYLOAD_BYTES.value + INDEX_K_SCALE_BYTES.value
@triton.jit
def _finish_paged_indexer_topk_kernel(
Indices,
Scores,
Lengths,
Requests,
ReqTable,
Pages,
Raw,
BATCH: tl.constexpr,
K: tl.constexpr,
WIDTH: tl.constexpr,
OUT_WIDTH: tl.constexpr,
IDX_STRIDE: tl.constexpr,
SCORE_STRIDE: tl.constexpr,
TABLE_STRIDE: tl.constexpr,
PAGE_STRIDE: tl.constexpr,
RAW_STRIDE: tl.constexpr,
RATIO: tl.constexpr,
MASK_SCORES: tl.constexpr,
WRITE_RAW: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
idx = tl.load(Indices + row * IDX_STRIDE + col, (row < BATCH) & (col < K), WIDTH)
if MASK_SCORES:
valid = (row < BATCH) & (col < K) & (idx >= 0) & (idx < WIDTH)
score = tl.load(Scores + row * SCORE_STRIDE + idx, valid, -float("inf"))
idx = tl.where(valid & (score > -float("inf")), idx, WIDTH)
idx = tl.sort(tl.where(idx >= 0, idx, WIDTH), descending=False)
length = tl.load(Lengths + row, row < BATCH, 0)
req = tl.load(Requests + row, row < BATCH, 0)
valid = (row < BATCH) & (col < K) & (idx < length)
slot = tl.load(
ReqTable + req.to(tl.int64) * TABLE_STRIDE + idx.to(tl.int64) * RATIO,
valid,
0,
)
tl.store(
Pages + row * PAGE_STRIDE + col,
tl.where(valid, slot.to(tl.int64) // RATIO, -1),
col < OUT_WIDTH,
)
if WRITE_RAW:
tl.store(
Raw + row * RAW_STRIDE + col, tl.where(valid, idx, -1), col < OUT_WIDTH
)
def finish_paged_indexer_topk(
indices: torch.Tensor,
scores: torch.Tensor,
lengths: torch.Tensor,
req: torch.Tensor,
req_table: torch.Tensor,
page_indices: torch.Tensor,
raw_indices: torch.Tensor | None,
ratio: int,
mask_scores: bool,
) -> None:
"""Sort selected positions and map them to compressed KV slots.
Candidate consumers discard selected masked scores, including top-k underfill.
The entire output (including padded rows/columns) is written, with -1 padding.
"""
if not page_indices.shape[0]:
return
_finish_paged_indexer_topk_kernel[(page_indices.shape[0],)](
indices,
scores,
lengths,
req,
req_table,
page_indices,
raw_indices,
lengths.numel(),
indices.shape[1],
scores.shape[1],
page_indices.shape[1],
indices.stride(0),
scores.stride(0),
req_table.stride(0),
page_indices.stride(0),
raw_indices.stride(0) if raw_indices is not None else 0,
ratio,
mask_scores,
raw_indices is not None,
triton.next_power_of_2(max(page_indices.shape[1], indices.shape[1])),
num_warps=4,
)
# Visible key counts change across batches; keep them out of the JIT key.
@triton.jit(do_not_specialize=["NUM_SLOTS"])
def _finish_flat_indexer_topk_kernel(
Indices,
Slots,
Starts,
Pages,
Raw,
K: tl.constexpr,
NUM_SLOTS,
IDX_STRIDE: tl.constexpr,
PAGE_STRIDE: tl.constexpr,
RAW_STRIDE: tl.constexpr,
WRITE_RAW: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
unselected = 2147483647
idx = tl.load(Indices + row * IDX_STRIDE + col, col < K, unselected)
idx = tl.sort(tl.where(idx < 0, unselected, idx), descending=False)
chosen = (col < K) & (idx != unselected)
slot = tl.load(Slots + tl.minimum(idx, NUM_SLOTS - 1), chosen, -1)
tl.store(Pages + row * PAGE_STRIDE + col, slot, col < K)
if WRITE_RAW:
start = tl.load(Starts + row)
tl.store(
Raw + row * RAW_STRIDE + col, tl.where(chosen, idx - start, -1), col < K
)
def finish_flat_indexer_topk(
indices: torch.Tensor,
slots: torch.Tensor,
request_starts: torch.Tensor,
page_indices: torch.Tensor,
raw_indices: torch.Tensor | None,
) -> None:
"""Sort flattened prefill positions and map them to KV slots in one kernel.
Negative positions and INT32_MAX are unselected. Write only the selected
rows and columns; callers retain responsibility for resetting output padding.
"""
rows, k = indices.shape
if not rows or not k:
return
assert indices.dtype == torch.int32 and indices.stride(1) == 1
assert slots.ndim == 1 and slots.is_contiguous() and slots.numel() > 0
assert request_starts.shape == (rows,) and request_starts.is_contiguous()
assert page_indices.shape[0] >= rows and page_indices.shape[1] >= k
assert page_indices.dtype == torch.int32 and page_indices.stride(1) == 1
if raw_indices is not None:
assert raw_indices.shape[0] >= rows and raw_indices.shape[1] >= k
assert raw_indices.dtype == torch.int32 and raw_indices.stride(1) == 1
_finish_flat_indexer_topk_kernel[(rows,)](
indices,
slots,
request_starts,
page_indices,
raw_indices,
k,
slots.numel(),
indices.stride(0),
page_indices.stride(0),
raw_indices.stride(0) if raw_indices is not None else 0,
raw_indices is not None,
triton.next_power_of_2(k),
num_warps=4,
)
@triton.jit
def _select_group_value(group, v0, v1, v2, v3):
return tl.where(
@@ -430,6 +594,73 @@ def _e2m1_decode(code):
return tl.where((code >> 3) == 1, -v, v)
@triton.jit
def _fp4_index_logits_tile(
q_ptr,
w_ptr,
table_ptr,
b,
slot,
valid,
page_size,
row_stride,
stride_qb,
stride_qh,
stride_wb,
H: tl.constexpr,
HALF_D: tl.constexpr,
):
offs_h = tl.arange(0, H)
offs_i = tl.arange(0, HALF_D)
page = slot // page_size
off = slot % page_size
row_base = page * row_stride
# K payload: [BLOCK_L, HALF_D] uint8
pay = tl.load(
table_ptr
+ row_base[:, None]
+ off[:, None] * INDEX_K_PAYLOAD_BYTES
+ offs_i[None, :],
mask=valid[:, None],
other=0,
)
low = _e2m1_decode(pay & 0x0F)
high = _e2m1_decode((pay >> 4) & 0x0F)
# e8m0 block scales: element j uses block j // 32 -> byte i uses block i // 16.
sc_idx = offs_i // 16
exps = tl.load(
table_ptr
+ row_base[:, None]
+ page_size * INDEX_K_PAYLOAD_BYTES
+ off[:, None] * INDEX_K_SCALE_BYTES
+ sc_idx[None, :],
mask=valid[:, None],
other=127,
)
scale = tl.exp2(exps.to(tl.float32) - 127.0)
k_low = (low * scale).to(tl.bfloat16) # [BLOCK_L, HALF_D] elements 2i
k_high = (high * scale).to(tl.bfloat16) # elements 2i+1
# queries: even / odd elements, [H, HALF_D] bf16
q_even = tl.load(
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
)
q_odd = tl.load(
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :] + 1
)
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
acc += tl.dot(q_odd, tl.trans(k_high))
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
s = acc.to(tl.bfloat16).to(tl.float32)
s = tl.maximum(s, 0.0)
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
logit = tl.where(valid, logit, float("-inf"))
return logit
@triton.jit
def _fp4_index_logits_kernel(
q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd)
@@ -456,11 +687,6 @@ def _fp4_index_logits_kernel(
b = tl.program_id(0)
lb = tl.program_id(1)
offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L)
offs_h = tl.arange(0, H)
offs_i = tl.arange(
0, HALF_D
) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble)
n_vis = tl.load(lens_ptr + b)
# Graph replay keeps the capacity-sized grid even for short live contexts.
# Skip whole invisible tiles using the current device-side length; masking
@@ -472,57 +698,21 @@ def _fp4_index_logits_kernel(
slot = tl.load(slots_ptr + b * L + offs_l, mask=offs_l < L, other=0).to(
tl.int64
)
page = slot // page_size
off = slot % page_size
row_base = page * row_stride
# K payload: [BLOCK_L, HALF_D] uint8
pay = tl.load(
table_ptr
+ row_base[:, None]
+ off[:, None] * INDEX_K_PAYLOAD_BYTES
+ offs_i[None, :],
mask=valid[:, None],
other=0,
logit = _fp4_index_logits_tile(
q_ptr,
w_ptr,
table_ptr,
b,
slot,
valid,
page_size,
row_stride,
stride_qb,
stride_qh,
stride_wb,
H,
HALF_D,
)
low = _e2m1_decode(pay & 0x0F)
high = _e2m1_decode((pay >> 4) & 0x0F)
# e8m0 block scales: element j uses block j // 32 -> byte i uses block i // 16.
sc_idx = offs_i // 16
exps = tl.load(
table_ptr
+ row_base[:, None]
+ page_size * INDEX_K_PAYLOAD_BYTES
+ off[:, None] * INDEX_K_SCALE_BYTES
+ sc_idx[None, :],
mask=valid[:, None],
other=127,
)
scale = tl.exp2(exps.to(tl.float32) - 127.0)
k_low = (low * scale).to(tl.bfloat16) # [BLOCK_L, HALF_D] elements 2i
k_high = (high * scale).to(tl.bfloat16) # elements 2i+1
# queries: even / odd elements, [H, HALF_D] bf16
q_even = tl.load(
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
)
q_odd = tl.load(
q_ptr
+ b * stride_qb
+ offs_h[:, None] * stride_qh
+ 2 * offs_i[None, :]
+ 1
)
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
acc += tl.dot(q_odd, tl.trans(k_high))
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
s = acc.to(tl.bfloat16).to(tl.float32)
s = tl.maximum(s, 0.0)
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
logit = tl.where(valid, logit, float("-inf"))
tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L)
@@ -569,3 +759,127 @@ def fp4_index_logits_decode(
num_warps=4,
)
return out
@triton.jit
def _fp4_index_logits_paged_kernel(
q_ptr,
w_ptr,
req_ptr,
req_table_ptr,
lens_ptr,
table_ptr,
mask_ptr,
out_ptr,
L,
page_size,
row_stride,
req_stride,
mask_stride,
out_stride,
stride_qb,
stride_qh,
stride_wb,
H: tl.constexpr,
HALF_D: tl.constexpr,
RATIO: tl.constexpr,
HAS_MASK: tl.constexpr,
BLOCK_L: tl.constexpr,
):
b = tl.program_id(0)
n_vis = tl.minimum(tl.load(lens_ptr + b), L)
req = tl.load(req_ptr + b).to(tl.int64)
for tile in range(tl.program_id(1), tl.cdiv(n_vis, BLOCK_L), tl.num_programs(1)):
offs_l = tile * BLOCK_L + tl.arange(0, BLOCK_L)
valid = offs_l < n_vis
if HAS_MASK:
valid = valid & tl.load(
mask_ptr + b * mask_stride + offs_l, mask=offs_l < n_vis, other=False
)
slot = (
tl.load(
req_table_ptr + req * req_stride + offs_l * RATIO, mask=valid, other=0
).to(tl.int64)
// RATIO
)
logit = _fp4_index_logits_tile(
q_ptr,
w_ptr,
table_ptr,
b,
slot,
valid,
page_size,
row_stride,
stride_qb,
stride_qh,
stride_wb,
H,
HALF_D,
)
tl.store(out_ptr + b * out_stride + offs_l, logit, mask=offs_l < L)
def fp4_index_logits_paged(
q: torch.Tensor,
weights: torch.Tensor,
req: torch.Tensor,
req_table: torch.Tensor,
lens: torch.Tensor,
table: torch.Tensor,
page_size: int,
max_len: int,
ratio: int,
candidate_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Score visible compressed positions directly from the request table.
The output has graph-stable capacity, but only the first ``lens[b]`` values
are initialized. Consumers must use the current device-side lengths.
The FP4 decoding and BF16 rounding are shared with the dense-slot path.
"""
assert q.dtype == torch.bfloat16 and q.shape[-1] == INDEX_HEAD_DIM
assert table.dtype == torch.uint8 and table.ndim == 2
assert ratio in (1, 2) and max_len <= req_table.shape[1] // ratio
assert req_table.stride(1) == 1
if candidate_mask is not None:
assert candidate_mask.shape[1] >= max_len and candidate_mask.stride(1) == 1
batch, heads, _ = q.shape
q = q.contiguous()
weights = weights.to(torch.bfloat16).contiguous()
req = req.contiguous()
lens = lens.contiguous()
# Existing candidate-amax needs rows aligned to eight floats.
out = torch.empty(
(batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32
)
if not batch or not max_len:
return out[:, :max_len]
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
workers = min(triton.cdiv(max_len, 64), triton.cdiv(num_sms * 4, batch))
_fp4_index_logits_paged_kernel[(batch, workers)](
q,
weights,
req,
req_table,
lens,
table,
candidate_mask,
out,
max_len,
page_size,
table.stride(0),
req_table.stride(0),
candidate_mask.stride(0) if candidate_mask is not None else 0,
out.stride(0),
q.stride(0),
q.stride(1),
weights.stride(0),
H=heads,
HALF_D=INDEX_HEAD_DIM // 2,
RATIO=ratio,
HAS_MASK=candidate_mask is not None,
BLOCK_L=64,
num_warps=4,
)
return out[:, :max_len]
@@ -5,11 +5,71 @@
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
"SPLIT_K": 16
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
@@ -5,10 +5,71 @@
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 8
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 2
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 1
},
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
@@ -1,15 +1,75 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"2": {
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
@@ -0,0 +1,80 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 8
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
}
}
@@ -0,0 +1,86 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 4
},
"33": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
},
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
}
}
@@ -0,0 +1,80 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"65": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3
}
}
+6 -3
View File
@@ -2611,7 +2611,10 @@ def hc_mix_stats_sinkhorn_bf16x3(
m, k = x.shape
mix = (2 + hc_mult) * hc_mult
slices = _HC_MIX_COMPENSATED_SLICES
assert x.is_contiguous() and x.dtype == torch.bfloat16 and 4096 <= m <= 65536
hopper_medium = get_platform().is_sm90 and 32 <= m < 4096
block_m = 64 if hopper_medium else _HC_MIX_BF16X3_BLOCK_M
assert x.is_contiguous() and x.dtype == torch.bfloat16
assert hopper_medium or 4096 <= m <= 65536
assert k % (slices * _HC_MIX_BLOCK_K) == 0
assert len(weight_parts) == 3
assert all(
@@ -2623,7 +2626,7 @@ def hc_mix_stats_sinkhorn_bf16x3(
pre = torch.empty((m, hc_mult), device=x.device, dtype=torch.float32)
post = torch.empty_like(pre)
comb = torch.empty((m, hc_mult, hc_mult), device=x.device, dtype=torch.float32)
_hc_mix_stats_bf16x3_kernel[(triton.cdiv(m, _HC_MIX_BF16X3_BLOCK_M), slices)](
_hc_mix_stats_bf16x3_kernel[(triton.cdiv(m, block_m), slices)](
x,
*weight_parts,
part_mix,
@@ -2634,7 +2637,7 @@ def hc_mix_stats_sinkhorn_bf16x3(
MIX_COLS=mix,
MIX_PAD=triton.next_power_of_2(mix),
BLOCK_K=_HC_MIX_BLOCK_K,
BLOCK_M=_HC_MIX_BF16X3_BLOCK_M,
BLOCK_M=block_m,
num_warps=4,
num_stages=3,
)
@@ -172,6 +172,7 @@ _TRITON_KERNELS = [
("fp8_kernel", "sglang_per_token_group_quant_8bit"),
("fp8_kernel", "sglang_per_token_quant_fp8"),
("fp8_kernel", "static_quant_fp8"),
("fp8_kernel", "dequant_group_fp8_to_bf16"),
("fp8_kernel", "per_tensor_quant_mla_fp8"),
("fp8_kernel", "per_token_group_quant_mla_deep_gemm_masked_fp8"),
("fp8_kernel", "per_token_group_quant_fp8_hopper_moe_mn_major"),
@@ -1389,6 +1389,25 @@ def per_group_transpose(
return trans_a
# input - [M, K]
# weight - [K, N]
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
@triton.jit
def _dequant_group_fp8_kernel(
X, S, Y, M, K: tl.constexpr, SS: tl.constexpr, BLOCK: tl.constexpr
):
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
rows, cols = offsets // K, offsets % K
value = tl.load(X + offsets, offsets < M * K, 0.0).to(tl.float32)
scale = tl.load(S + rows * SS + cols // 32, rows < M, 0.0)
tl.store(Y + offsets, value * scale, offsets < M * K)
def dequant_group_fp8_to_bf16(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
"""Expand row-wise group32 FP8 values; UE8M0 scaling is exact in BF16."""
m, k = x.shape
assert x.is_contiguous() and k % 32 == 0
assert scale.shape == (m, k // 32) and scale.stride(1) == 1
out = torch.empty_like(x, dtype=torch.bfloat16)
if m:
_dequant_group_fp8_kernel[(triton.cdiv(m * k, 1024),)](
x, scale, out, m, k, scale.stride(0), 1024
)
return out
+3
View File
@@ -1015,6 +1015,9 @@ class Envs:
SGLANG_CPU_QUANTIZATION = EnvBool(False)
SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False)
SGLANG_FORCE_FP8_MARLIN = EnvBool(False)
# Cache BF16 expansions of Hopper group32 FP8 weights for larger GEMMs.
# Disable to save the additional weight memory or allow online updates.
SGLANG_OPT_HOPPER_BLOCK_FP8_BF16 = EnvBool(True)
SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False)
SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN = EnvBool(False)
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False)
@@ -9,16 +9,20 @@ import msgspec
import torch
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
amax_topk_blocks,
candidate_block_mask,
select_candidate_block_ids,
topk_among_blocks,
)
from .scoring import (
DeepGEMMPrefillData,
PagedDecodeScores,
decode_scores,
get_deep_gemm_prefill_data,
prefill_requests,
score_tiles,
select_decode,
write_decode,
write_prefill,
)
@@ -40,6 +44,8 @@ class BlockIds(CandidateMetadata, msgspec.Struct):
blocks: torch.Tensor
# prefill: query rows of each request in row order, for the late-layer tail
rows_per_request: Optional[List[int]] = None
# Hopper decode materializes this once at the source and reuses it in consumers.
decode_mask: Optional[torch.Tensor] = None
def tail(self, rows_per_request: List[int]) -> BlockIds:
assert self.rows_per_request is not None, "prefill block ids missing"
@@ -92,17 +98,25 @@ class DenseBlocksBackend:
)
if d is None:
return None
published = BlockIds(
blocks=select_candidate_block_ids(
d.scores,
d.lens[:, None],
topk_blocks=self.topk_blocks,
block_size=self.block_size,
if isinstance(d, PagedDecodeScores):
assert self.block_size == 8
blocks = amax_topk_blocks(
d.scores, d.lens, (d.lens + 7) // 8, self.topk_blocks
)
)
k = min(inputs.indexer.index_topk, d.lmax)
idx = d.scores.topk(k, dim=-1, sorted=False).indices
write_decode(out, d, idx)
published = BlockIds(
blocks=blocks,
decode_mask=candidate_block_mask(blocks, d.lmax, self.block_size),
)
else:
published = BlockIds(
blocks=select_candidate_block_ids(
d.scores,
d.lens[:, None],
topk_blocks=self.topk_blocks,
block_size=self.block_size,
)
)
select_decode(out, d, inputs.indexer.index_topk)
return published
def consume_decode(
@@ -116,10 +130,15 @@ class DenseBlocksBackend:
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
candidate_mask=published.decode_mask if published is not None else None,
)
if d is None:
return
assert published is not None and published.blocks.shape[0] == d.bs
if isinstance(d, PagedDecodeScores):
assert published.decode_mask is not None
select_decode(out, d, inputs.indexer.index_topk)
return
k = min(inputs.indexer.index_topk, d.lmax)
idx = topk_among_blocks(
d.scores, d.lens, published.blocks, k, block_size=self.block_size
@@ -21,7 +21,7 @@ from .scoring import (
get_index_k_cache,
prefill_requests,
quantize_index_q,
write_decode,
select_decode,
write_prefill,
)
from .types import (
@@ -138,9 +138,7 @@ class FullTopKIndexer:
)
if d is None:
return
k = min(inputs.indexer.index_topk, d.lmax)
idx = d.scores.topk(k, dim=-1, sorted=False).indices
write_decode(out, d, idx)
select_decode(out, d, inputs.indexer.index_topk)
def _deep_gemm_prefill_captured(
self, inputs: CapturedPrefillInputs, out: Selection
@@ -9,8 +9,18 @@ from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
import msgspec
import torch
from sglang.kernels.ops.attention.dsv4.fp4_indexer import fp4_index_logits_decode
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
finish_flat_indexer_topk,
finish_paged_indexer_topk,
fp4_index_logits_decode,
fp4_index_logits_paged,
)
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
from sglang.kernels.ops.attention.dsv4.topk import (
plan_topk_v2,
topk_transform_paged_v2,
)
from sglang.srt.runtime_context import get_platform
from .types import (
DecodeInputs,
@@ -57,6 +67,19 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
)
def write_selection(self, selected: torch.Tensor, out: Selection) -> None:
if (
selected.is_cuda
and get_platform().is_sm100
and selected.dtype == torch.int32
):
finish_flat_indexer_topk(
selected,
self.k_slots,
self.request_starts,
out.page_indices,
out.raw_indices,
)
return
num_tokens, topk = selected.shape
unselected = torch.iinfo(torch.int32).max
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
@@ -332,24 +355,45 @@ def write_prefill(
out.raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
class PagedDecodeScores(msgspec.Struct, frozen=True):
"""Paged decode scores with graph-stable capacity ``lmax``.
Only positions below each device-side ``lens`` are initialized. Top-k
consumers must respect those lengths. With ``has_candidate_mask``, masked
positions have -inf scores and must be discarded during selection writeback.
"""
bs: int
lmax: int
lens: torch.Tensor
scores: torch.Tensor
req: torch.Tensor
req_to_token: torch.Tensor
ratio: int
plan: torch.Tensor
has_candidate_mask: bool
def decode_scores(
*,
inputs: DecodeInputs,
out: Selection,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
) -> Optional[DecodeScores]:
candidate_mask: Optional[torch.Tensor] = None,
) -> Optional[DecodeScores | PagedDecodeScores]:
"""None when there is no row, or nothing visible yet."""
pool = token_to_kv_pool
ratio = inputs.compress_ratio
indexer = inputs.indexer
req, pos = inputs.req_rows, inputs.positions
out.reset()
paged = get_platform().is_sm90
bs = req.shape[0]
assert pos.shape[0] == bs, (
f"decode expects one token per request, {pos.shape=} {bs=}"
)
if bs == 0:
out.reset()
return None
lens = (pos + 1) // ratio
metadata = inputs.paged_metadata
@@ -357,20 +401,71 @@ def decode_scores(
# A capture-time length read would both synchronize and truncate replay.
lmax = min(metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio)
if lmax == 0:
out.reset()
return None
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
table = pool.get_index_k_with_scale_buffer(inputs.layer_id)
if paged:
lens = lens.to(torch.int32)
# Prepare the device-side plan before scoring, away from its top-k consumer.
plan = plan_topk_v2(lens)
scores = fp4_index_logits_paged(
q,
weights,
req,
req_to_token,
lens,
table,
table.shape[1] // 68,
lmax,
ratio,
candidate_mask,
)
return PagedDecodeScores(
bs=bs,
lmax=lmax,
lens=lens,
scores=scores,
req=req,
req_to_token=req_to_token,
ratio=ratio,
plan=plan,
has_candidate_mask=candidate_mask is not None,
)
out.reset()
j = torch.arange(lmax, device=pos.device)
valid = j[None, :] < lens[:, None]
slots = req_to_token[req[:, None], (j * ratio)[None, :]].to(torch.int64) // ratio
slots = slots.masked_fill(~valid, 0)
table = pool.get_index_k_with_scale_buffer(inputs.layer_id)
scores = fp4_index_logits_decode(
q, weights, slots, lens, table, table.shape[1] // 68
)
return DecodeScores(bs=bs, lmax=lmax, lens=lens, slots=slots, scores=scores)
def select_decode(
out: Selection, d: DecodeScores | PagedDecodeScores, topk: int
) -> None:
k = min(topk, d.lmax)
if isinstance(d, PagedDecodeScores):
idx = torch.empty((d.bs, k), dtype=torch.int32, device=d.scores.device)
topk_transform_paged_v2(d.scores, d.lens, None, idx, 1, d.plan)
finish_paged_indexer_topk(
idx,
d.scores,
d.lens,
d.req,
d.req_to_token,
out.page_indices,
out.raw_indices,
d.ratio,
d.has_candidate_mask,
)
else:
write_decode(out, d, d.scores.topk(k, dim=-1, sorted=False).indices)
def write_decode(out: Selection, d: DecodeScores, idx: torch.Tensor) -> None:
k = idx.shape[1]
idx = idx.sort(dim=-1).values
@@ -6,6 +6,7 @@ import triton
import triton.language as tl
from sglang.srt.layers import zero_copy_context
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils import is_cuda
from sglang.srt.utils.custom_op import register_custom_op
@@ -14,7 +15,10 @@ _is_cuda = is_cuda()
if _is_cuda:
from sgl_kernel import moe_sum_reduce
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.activation.activation import (
silu_and_mul,
silu_and_mul_with_activation_rounding,
)
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
@@ -109,6 +113,17 @@ def swiglu_limit_func(
swiglu_limit: float = 0.0,
) -> None:
d = input.shape[1] // 2
if (
_is_cuda
and get_platform().is_sm90
and input.is_cuda
and input.dtype in (torch.bfloat16, torch.float16)
and d % 16 == 0
and input.is_contiguous()
and output.is_contiguous()
):
silu_and_mul_with_activation_rounding(input, output, clamp_limit=swiglu_limit)
return
gate = input[:, :d]
up = input[:, d:]
@@ -819,6 +819,46 @@ class Fp8LinearMethod(LinearMethodBase):
layer.weight.data = weight.data
layer.weight_scale_inv.data = weight_scale.data
if hasattr(layer, "_block_fp8_bf16_weight"):
layer._block_fp8_bf16_weight = None
layer._derived_weight_cache_error = None
if (
_is_cuda
and get_platform().is_sm90
and envs.SGLANG_OPT_HOPPER_BLOCK_FP8_BF16.get()
and weight.is_cuda
and weight.dtype == torch.float8_e4m3fn
and self.weight_block_size == [32, 32]
and getattr(self.quant_config, "scale_fmt", None) == "ue8m0"
and getattr(layer, "orig_dtype", None) == torch.bfloat16
and not self.use_marlin
and not getattr(layer, "keep_plain_weight_layout", False)
):
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.quantization.fp8_utils import block_quant_dequant
# Keep even the smallest/largest finite E4M3 value representable
# as normal BF16, and reject mislabeled non-power-of-two scales.
exact_bf16_scales = (
(weight_scale >= 2.0**-117)
& (weight_scale <= 2.0**119)
& (torch.frexp(weight_scale)[0] == 0.5)
).all()
if not is_batch_invariant_mode_enabled() and exact_bf16_scales.item():
# Hopper group32 FP8 accumulates each 32-wide block separately.
# Reuse a BF16 weight expansion for larger GEMMs, keeping the
# original FP8 weights for decode and weight reloads.
layer.register_buffer(
"_block_fp8_bf16_weight",
block_quant_dequant(weight, weight_scale, [32, 32], torch.bfloat16),
persistent=False,
)
layer._derived_weight_cache_error = (
"Online weight updates are not supported while Hopper FP8 "
"BF16 weight caches are active: captured CUDA graphs retain "
"these derived weights. Restart with "
"SGLANG_OPT_HOPPER_BLOCK_FP8_BF16=0 to allow online updates."
)
if self.block_fp8_as_mxfp8:
self._prepare_block_fp8_as_mxfp8(layer)
@@ -1233,6 +1273,10 @@ class Fp8LinearMethod(LinearMethodBase):
True, # is_vnni
)
cached_weight = getattr(layer, "_block_fp8_bf16_weight", None)
extra_kwargs = (
{"weight_bf16": cached_weight} if cached_weight is not None else {}
)
if isinstance(x, tuple):
return self.w8a8_block_fp8_linear(
input=x[0],
@@ -1241,6 +1285,7 @@ class Fp8LinearMethod(LinearMethodBase):
weight_scale=layer.weight_scale_inv,
input_scale=x[1],
bias=bias,
**extra_kwargs,
)
return self.w8a8_block_fp8_linear(
@@ -1250,6 +1295,7 @@ class Fp8LinearMethod(LinearMethodBase):
weight_scale=layer.weight_scale_inv,
input_scale=None,
bias=bias,
**extra_kwargs,
)
if use_intel_amx_backend(layer):
@@ -14,6 +14,7 @@ from sglang.kernels.ops.gemm.fp8_kernel import (
w8a8_block_fp8_matmul_triton,
)
from sglang.kernels.ops.quantization.fp8_kernel import (
dequant_group_fp8_to_bf16,
fp8_dtype,
fp8_max,
fp8_min,
@@ -1430,6 +1431,7 @@ def triton_w8a8_block_fp8_linear(
input_scale: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
act_scale_ue8m0: bool = False,
weight_bf16: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if input_scale is not None:
# Pre-quantized input: ``input`` is already fp8 and ``input_scale`` is
@@ -1454,9 +1456,30 @@ def triton_w8a8_block_fp8_linear(
input_2d, block_size[1], column_major_scales=False
)
output = w8a8_block_fp8_matmul_triton(
q_input, weight, x_scale, weight_scale, block_size, output_dtype=output_dtype
)
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
if (
weight_bf16 is not None
and input_scale is None
and act_scale_ue8m0
and block_size == [32, 32]
and q_input.shape[0] >= 64
and output_dtype == torch.bfloat16
and not is_batch_invariant_mode_enabled()
):
# Keep activation quantization: only the GEMM implementation changes.
output = torch.nn.functional.linear(
dequant_group_fp8_to_bf16(q_input, x_scale), weight_bf16
)
else:
output = w8a8_block_fp8_matmul_triton(
q_input,
weight,
x_scale,
weight_scale,
block_size,
output_dtype=output_dtype,
)
if bias is not None:
output += bias
return output.to(dtype=output_dtype).view(*output_shape)
@@ -71,7 +71,13 @@ class Mxfp4MarlinMoEMethod:
layer._dsv4_mxfp4_backend = None # set in process_weights_after_loading
fp4_block_k = 32
intermediate_size_per_partition = round_up(intermediate_size_per_partition, 128)
# Hopper repacking pads the loaded gate/up halves and down columns to
# the required tile sizes. Keep their logical widths until then: early
# padding to 128 turns a TP8 width of 288 into 384 instead of 320.
if not get_platform().is_sm90:
intermediate_size_per_partition = round_up(
intermediate_size_per_partition, 128
)
hidden_size = round_up(hidden_size, 256)
self.hidden_pad = hidden_size - layer.hidden_size
@@ -42,14 +42,18 @@ def _unsupported_derived_weight_cache_error(
) -> Optional[str]:
"""Reject online weight updates that derived-weight caches cannot survive.
The HPC-Ops bf16xfp32 GEMM caches the fp32 weight split; in-place loader
writes are invisible to it, so an update would silently keep serving the
old weights. The check is startup-determined and rank-uniform, so an
update never proceeds on some workers while rejected on others.
Compensated mHC projections and HPC-Ops bf16xfp32 GEMM retain derived
weight splits that in-place loader writes do not refresh. Model-owned
caches can also declare this constraint via ``_derived_weight_cache_error``.
Reject before writing weights to avoid serving stale cached values,
including references retained by captured CUDA graphs. These constraints
must be startup-determined and rank-uniform so all workers reject together.
"""
if model is not None and any(
getattr(module, "_hc_attn_tf32_parts", None) is not None
or getattr(module, "_hc_ffn_tf32_parts", None) is not None
or getattr(module, "_hc_attn_bf16_parts", None) is not None
or getattr(module, "_hc_ffn_bf16_parts", None) is not None
for module in model.modules()
):
return (
+113 -76
View File
@@ -2726,12 +2726,21 @@ class DeepseekV4DecoderLayer(nn.Module):
self._hc_attn_bf16_parts = self._hc_ffn_bf16_parts = None
if (
self.hc_pre_from_prev_sublayer
and get_platform().is_sm100
and (get_platform().is_sm100 or get_platform().is_sm90)
and self.hc_attn_fn.shape == (24, 20480)
and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get()
and getattr(self.config, "model_type", None) == "deepseek_v41"
and not is_batch_invariant_mode_enabled()
):
if get_platform().is_sm90:
from sglang.kernels.ops.layernorm.mhc import split_bf16_hc_weight
# The compensated BF16 projection uses ordinary tensor cores;
# it does not require Blackwell or DeepGEMM's prenorm kernel.
self._hc_attn_bf16_parts = split_bf16_hc_weight(self.hc_attn_fn.data)
self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data)
return
from sglang.kernels.ops.layernorm.mhc import (
split_tf32_hc_weight,
)
@@ -3237,7 +3246,10 @@ class DeepseekV4DecoderLayer(nn.Module):
if (
x.is_cuda
and get_platform().is_blackwell
and (
get_platform().is_blackwell
or (get_platform().is_sm90 and not quantize)
)
and (
0 < x.shape[0] <= 96
or (
@@ -3283,90 +3295,107 @@ class DeepseekV4DecoderLayer(nn.Module):
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
stats_stream: Optional[torch.cuda.Stream] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
if stats_stream is None:
return DeepseekV4DecoderLayer._hc_mix_stats_impl(
self, x, hc_fn, hc_scale, hc_base
)
main_stream = torch.cuda.current_stream()
x.record_stream(stats_stream)
with torch.cuda.stream(stats_stream):
coefficients = DeepseekV4DecoderLayer._hc_mix_stats_impl(
self, x, hc_fn, hc_scale, hc_base
)
# The producer fork is in _hc_combine; consumers join before hc_post.
for coefficient in coefficients:
coefficient.record_stream(main_stream)
return coefficients
def _hc_mix_stats_impl(
self,
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
from sglang.kernels.ops.layernorm.mhc import hc_mix_stats, hc_mix_stats_sinkhorn
x_flat = x.flatten(1)
from sglang.srt.batch_invariant_ops import (
is_batch_invariant_mode_enabled,
)
parts = bf16_parts = None
hopper_medium = get_platform().is_sm90 and 32 <= x_flat.shape[0] < 4096
if (
x.is_cuda
and torch.version.cuda is not None
and (
get_platform().is_blackwell
or (get_platform().is_sm90 and x.shape[0] == 1)
)
and x.dtype == torch.bfloat16
and (x_flat.shape[0] >= 128 or hopper_medium)
and x_flat.is_contiguous()
and (get_platform().is_sm100 or get_platform().is_sm90)
and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get()
and not is_batch_invariant_mode_enabled()
):
# Fusing the split-K reduction with sinkhorn keeps it batch-invariant.
main_stream = torch.cuda.current_stream()
if stats_stream is not None:
x.record_stream(stats_stream)
with (
torch.cuda.stream(stats_stream)
if stats_stream is not None
else nullcontext()
):
from sglang.srt.batch_invariant_ops import (
is_batch_invariant_mode_enabled,
if hc_fn is self.hc_attn_fn:
parts = getattr(self, "_hc_attn_tf32_parts", None)
bf16_parts = getattr(self, "_hc_attn_bf16_parts", None)
elif hc_fn is self.hc_ffn_fn:
parts = getattr(self, "_hc_ffn_tf32_parts", None)
bf16_parts = getattr(self, "_hc_ffn_bf16_parts", None)
use_bf16_projection = bf16_parts is not None and (
hopper_medium or 4096 <= x_flat.shape[0] <= 65536
)
hopper_fused_stats = get_platform().is_sm90 and (
x.shape[0] == 1 or (bf16_parts is not None and 32 <= x.shape[0] <= 65536)
)
# gfx950 uses the fused Triton port at every row count (MI350X).
use_fused_stats = _is_gfx95_supported or (
torch.version.cuda is not None
and (get_platform().is_blackwell or hopper_fused_stats)
)
if x.is_cuda and use_fused_stats and x.dtype == torch.bfloat16:
# The default split-K/Sinkhorn fusion preserves batch invariance;
# compensated projections above are disabled in batch-invariant mode.
if use_bf16_projection:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_bf16x3,
)
parts = bf16_parts = None
if (
x_flat.shape[0] >= 128
and x_flat.is_contiguous()
and get_platform().is_sm100
and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get()
and not is_batch_invariant_mode_enabled()
):
if hc_fn is self.hc_attn_fn:
parts = getattr(self, "_hc_attn_tf32_parts", None)
bf16_parts = getattr(self, "_hc_attn_bf16_parts", None)
elif hc_fn is self.hc_ffn_fn:
parts = getattr(self, "_hc_ffn_tf32_parts", None)
bf16_parts = getattr(self, "_hc_ffn_bf16_parts", None)
if bf16_parts is not None and 4096 <= x_flat.shape[0] <= 65536:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_bf16x3,
)
pre, post, comb = hc_mix_stats_sinkhorn_bf16x3(
x_flat,
bf16_parts,
hc_scale,
hc_base,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
elif parts is not None:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_deepgemm,
)
pre, post, comb = hc_mix_stats_sinkhorn_bf16x3(
x_flat,
bf16_parts,
hc_scale,
hc_base,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
elif parts is not None:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_deepgemm,
)
pre, post, comb = hc_mix_stats_sinkhorn_deepgemm(
x_flat,
parts,
hc_scale,
hc_base,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
else:
pre, post, comb = hc_mix_stats_sinkhorn(
x_flat,
hc_fn,
hc_scale,
hc_base,
self.hc_mult,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
if stats_stream is not None:
# Allocated on the side stream, read on the main stream after the join.
for coefficient in (pre, post, comb):
coefficient.record_stream(main_stream)
pre, post, comb = hc_mix_stats_sinkhorn_deepgemm(
x_flat,
parts,
hc_scale,
hc_base,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
else:
pre, post, comb = hc_mix_stats_sinkhorn(
x_flat,
hc_fn,
hc_scale,
hc_base,
self.hc_mult,
self.hc_sinkhorn_iters,
self.rms_norm_eps,
self.hc_eps,
)
return pre, post, comb
if x.is_cuda and torch.version.cuda is not None:
# cuBLAS/torch reductions can change order with num_tokens; this kernel
@@ -3439,7 +3468,15 @@ class DeepseekV4DecoderLayer(nn.Module):
and hidden_states.shape[0] > 0
)
)
and (not get_platform().is_sm90 or hidden_states.shape[0] == 1)
and (
not get_platform().is_sm90
or hidden_states.shape[0] == 1
or (
self.config.model_type == "deepseek_v41"
and forward_batch.forward_mode.is_decode()
and 1 < hidden_states.shape[0] <= 64
)
)
else None
)