diff --git a/python/sglang/kernels/ops/activation/__init__.py b/python/sglang/kernels/ops/activation/__init__.py index 5dbd6865ac..14359b322a 100644 --- a/python/sglang/kernels/ops/activation/__init__.py +++ b/python/sglang/kernels/ops/activation/__init__.py @@ -344,6 +344,20 @@ for _fn in ("softcap_out", "softcap_inplace_logits"): del _fn +register_kernel( + KernelSpec( + op="activation.softcap_to_float32_logits", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits", + capabilities=_CUDA, + format_signature=FormatSignature( + supported_dtypes=("float16", "bfloat16", "float32"), + description="Softcap logits into a caller-provided FP32 output.", + ), + ) +) + + # Kernels introduced with Kimi-K3, inventoried by logical operator group. register_kernel( KernelSpec( diff --git a/python/sglang/kernels/ops/activation/softcap.py b/python/sglang/kernels/ops/activation/softcap.py index 8b719c9b9a..317dfed36c 100644 --- a/python/sglang/kernels/ops/activation/softcap.py +++ b/python/sglang/kernels/ops/activation/softcap.py @@ -75,6 +75,9 @@ def softcap_inplace_logits_kernel( ncols, row_stride, BLOCK_SIZE: tl.constexpr, + output_ptr=None, + output_row_stride=0, + CAST_TO_FP32: tl.constexpr = False, ): row = tl.program_id(1).to(tl.int64) pid = tl.program_id(0).to(tl.int64) @@ -85,18 +88,24 @@ def softcap_inplace_logits_kernel( # Load values row_ptr = full_logits_ptr + row * row_stride x = tl.load(row_ptr + offsets, mask=mask) + if CAST_TO_FP32: + x = x.to(tl.float32) - # Perform operations in-place x = x / softcapping_value x = libdevice.tanh(x) x = x * softcapping_value # Store result - tl.store(row_ptr + offsets, x, mask=mask) + if CAST_TO_FP32: + tl.store(output_ptr + row * output_row_stride + offsets, x, mask=mask) + else: + tl.store(row_ptr + offsets, x, mask=mask) -def softcap_inplace_logits(full_logits, final_logit_softcapping): - if full_logits.is_contiguous(): +def _softcap_logits(full_logits, final_logit_softcapping, output=None): + if output is not None: + assert output.shape == full_logits.shape and output.dtype == torch.float32 + if full_logits.is_contiguous() and (output is None or output.is_contiguous()): nrows, ncols = 1, full_logits.numel() row_stride = ncols else: @@ -106,6 +115,10 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping): ) nrows, ncols = full_logits.shape row_stride = full_logits.stride(0) + if output is not None: + assert output.stride(1) == 1 and ( + nrows <= 1 or output.stride(0) >= ncols + ), "softcap output requires contiguous, non-overlapping rows" BLOCK_SIZE = 1024 grid = ((ncols + BLOCK_SIZE - 1) // BLOCK_SIZE, nrows) @@ -116,5 +129,19 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping): ncols=ncols, row_stride=row_stride, BLOCK_SIZE=BLOCK_SIZE, + output_ptr=output, + output_row_stride=(ncols if nrows == 1 else output.stride(0)) + if output is not None + else 0, + CAST_TO_FP32=output is not None, ) - return full_logits + return full_logits if output is None else output + + +def softcap_inplace_logits(full_logits, final_logit_softcapping): + return _softcap_logits(full_logits, final_logit_softcapping) + + +def softcap_to_float32_logits(logits, final_logit_softcapping, output): + """Write softcapped FP32 logits to an equally shaped, row-contiguous output.""" + return _softcap_logits(logits, final_logit_softcapping, output) diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index c2e046daa8..3e8b66f5f2 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -360,6 +360,43 @@ register_kernel( ) ) +register_kernel( + KernelSpec( + op="attention.pack_prefix_current", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current", + capabilities=frozenset({CapabilityRequirement.CUDA}), + description="Pack CSR prefix and current K/V into caller-provided dense buffers.", + ) +) +register_kernel( + KernelSpec( + op="attention.flash_attn_gqa_512", + backend=KernelBackend.CUTE_DSL, + target="sglang.kernels.ops.attention.flash_attention_v4:flash_attn_gqa_512", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))} + ), + format_signature=FormatSignature( + supported_dtypes=("float16", "bfloat16"), + description="SM100/SM110 D512 GQA with separate K/V and optional page tables.", + ), + ) +) +register_kernel( + KernelSpec( + op="attention.gemma_qkv_norm_rope", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope", + capabilities=frozenset({CapabilityRequirement.CUDA}), + format_signature=FormatSignature( + supported_dtypes=("float16", "bfloat16"), + in_place=True, + description="Strided QKV normalization and partial NeoX RoPE.", + ), + ) +) + for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"): register_kernel( KernelSpec( diff --git a/python/sglang/kernels/ops/attention/dllm_kv_pack.py b/python/sglang/kernels/ops/attention/dllm_kv_pack.py new file mode 100644 index 0000000000..30b9a5a72d --- /dev/null +++ b/python/sglang/kernels/ops/attention/dllm_kv_pack.py @@ -0,0 +1,93 @@ +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0. +"""Pack a CSR prefix and current K/V into capture-stable dense storage.""" + +import triton +import triton.language as tl + + +@triton.jit +def _pack_kv( + K, + V, + KB, + VB, + QO, + KI, + IDS, + KD, + VD, + CU, + H: tl.constexpr, + D: tl.constexpr, + KS: tl.constexpr, + KH: tl.constexpr, + VS: tl.constexpr, + VH: tl.constexpr, + KBS: tl.constexpr, + KBH: tl.constexpr, + VBS: tl.constexpr, + VBH: tl.constexpr, + TILE: tl.constexpr, +): + row = tl.program_id(0) + qs = tl.load(QO + row).to(tl.int64) + qe = tl.load(QO + row + 1).to(tl.int64) + ps = tl.load(KI + row).to(tl.int64) + pe = tl.load(KI + row + 1).to(tl.int64) + prefix = pe - ps + length = prefix + qe - qs + start = ps + qs + offsets = tl.program_id(1).to(tl.int64) * TILE + tl.arange(0, TILE) + token, head, channel = offsets // (H * D), offsets // D % H, offsets % D + slot = tl.load(IDS + ps + token, token < prefix, other=0).to(tl.int64) + kp = tl.load(KB + slot * KBS + head * KBH + channel, token < prefix, other=0) + vp = tl.load(VB + slot * VBS + head * VBH + channel, token < prefix, other=0) + current = qs + token - prefix + kc = tl.load( + K + current * KS + head * KH + channel, + (token >= prefix) & (token < length), + other=0, + ) + vc = tl.load( + V + current * VS + head * VH + channel, + (token >= prefix) & (token < length), + other=0, + ) + tl.store( + KD + start * H * D + offsets, tl.where(token < prefix, kp, kc), token < length + ) + tl.store( + VD + start * H * D + offsets, tl.where(token < prefix, vp, vc), token < length + ) + if tl.program_id(1) == 0: + tl.store(CU + row, start) + if row == tl.num_programs(0) - 1: + tl.store(CU + row + 1, pe + qe) + + +def pack_prefix_current(k, v, kb, vb, qo, ki, ids, kd, vd, cu, max_length): + h, d = k.shape[1:] + _pack_kv[(qo.numel() - 1, triton.cdiv(max_length * h * d, 1024))]( + k, + v, + kb, + vb, + qo, + ki, + ids, + kd, + vd, + cu, + h, + d, + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + kb.stride(0), + kb.stride(1), + vb.stride(0), + vb.stride(1), + TILE=1024, + ) diff --git a/python/sglang/kernels/ops/attention/extend_attention.py b/python/sglang/kernels/ops/attention/extend_attention.py index 38606d936d..94e8f43989 100644 --- a/python/sglang/kernels/ops/attention/extend_attention.py +++ b/python/sglang/kernels/ops/attention/extend_attention.py @@ -27,12 +27,7 @@ from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors from sglang.srt.environ import envs -from sglang.srt.utils import ( - is_cuda, - is_gfx95_supported, - is_gfx1250_supported, - is_hip, -) +from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip _is_cuda = is_cuda() if _is_cuda: @@ -1257,8 +1252,14 @@ def _fwd_kernel_unified( deno = tl.zeros([BLOCK_M], dtype=tl.float32) e_max = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") - # Unified loop: process all KV tokens (prefix + extend) - for start_n in range(0, cur_seq_kv_len, BLOCK_N): + start_k, end_k = 0, cur_seq_kv_len + if IS_CAUSAL and SLIDING_WINDOW_SIZE > 0 and not USE_CUSTOM_MASK: + first_query = cur_block_m * BLOCK_M + last_query = tl.minimum(first_query + BLOCK_M, cur_seq_q_len) + start_k = tl.maximum(0, cur_seq_prefix_len + first_query - SLIDING_WINDOW_SIZE) + start_k = start_k // BLOCK_N * BLOCK_N + end_k = tl.minimum(cur_seq_kv_len, cur_seq_prefix_len + last_query) + for start_n in range(start_k, end_k, BLOCK_N): start_n = tl.multiple_of(start_n, BLOCK_N) mask_n = (start_n + offs_n) < cur_seq_kv_len diff --git a/python/sglang/kernels/ops/attention/flash_attention_v4.py b/python/sglang/kernels/ops/attention/flash_attention_v4.py index 499b956d22..3a986fd2d4 100644 --- a/python/sglang/kernels/ops/attention/flash_attention_v4.py +++ b/python/sglang/kernels/ops/attention/flash_attention_v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from functools import lru_cache from typing import Callable, Optional, Tuple, Union import torch @@ -28,6 +29,102 @@ def is_flash_attention_v4_available() -> bool: return _flash_attn_varlen_func is not None +@lru_cache(maxsize=1) +def _get_gqa_512_jit_cache(): + from sglang.kernels.ops.attention.flash_attn.cute.interface import _get_jit_cache + + return _get_jit_cache("fwd_gqa_512") + + +def flash_attn_gqa_512( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + out: torch.Tensor, + *, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + softmax_scale: float = 1.0, + lse: Optional[torch.Tensor] = None, + pack_gqa: bool = True, + causal: bool = False, +) -> torch.Tensor: + """GQA with 512-dimensional keys and separate values.""" + import cutlass.cute as cute + + from sglang.kernels.ops.attention.flash_attn.cute.cute_dsl_utils import ( + to_cute_tensor, + ) + from sglang.kernels.ops.attention.flash_attn.cute.flash_fwd_mla_sm100 import ( + FlashAttentionMLAForwardSm100, + ) + + if cu_seqlens_q is not None: + cu_seqlens_q = cu_seqlens_q.to(dtype=torch.int32) + args = ( + q, + k, + v, + out, + lse, + softmax_scale, + cu_seqlens_q, + cu_seqlens_k, + seqused_k, + page_table, + ) + use_cpasync = page_table is not None and k.shape[1] != 128 + key = ( + pack_gqa, + causal, + q.device, + q.shape[-2] // k.shape[-2], + k.shape[-2], + q.shape[-1], + v.shape[-1], + use_cpasync, + tuple( + (t.ndim, t.dtype, tuple(s if s in (0, 1) else 2 for s in t.stride())) + if isinstance(t, torch.Tensor) + else t + for t in args + ), + ) + cache = _get_gqa_512_jit_cache() + if key not in cache: + compile_args = [ + to_cute_tensor( + t, + assumed_align=4 + if t.dtype in (torch.int32, torch.int64, torch.float32) + else 16, + ) + if isinstance(t, torch.Tensor) + else t + for t in args + ] + kernel = FlashAttentionMLAForwardSm100( + is_causal=causal, + use_cpasync_load_KV=use_cpasync, + is_topk_gather=False, + pack_gqa=pack_gqa, + qhead_per_kvhead=q.shape[-2] // k.shape[-2], + nheads_kv=k.shape[-2], + is_varlen_q=cu_seqlens_q is not None, + has_qk=False, + ) + cache[key] = cute.compile( + kernel.forward_gqa, + *compile_args, + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + cache[key](*args) + return out + + def _maybe_contiguous(x: Optional[torch.Tensor]) -> Optional[torch.Tensor]: return x.contiguous() if x is not None and x.stride(-1) != 1 else x diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py index 40b0b69673..3aa7e8e5dd 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py @@ -353,6 +353,28 @@ class FlashAttentionMLAForwardSm100: return SharedStorage + @cute.jit + def forward_gqa( + self, + mQ: cute.Tensor, + mK: cute.Tensor, + mValue: cute.Tensor, + mO: cute.Tensor, + mLSE: Optional[cute.Tensor], + softmax_scale: Float32, + mCuSeqlensQ: Optional[cute.Tensor] = None, + mCuSeqlensK: Optional[cute.Tensor] = None, + mSeqUsedK: Optional[cute.Tensor] = None, + mPageTable: Optional[cute.Tensor] = None, + stream: cuda.CUstream = None, + ): + self( + None, mQ, None, mK, mO, mLSE, softmax_scale, + mCuSeqlensQ=mCuSeqlensQ, mCuSeqlensK=mCuSeqlensK, + mSeqUsedK=mSeqUsedK, mPageTable=mPageTable, + stream=stream, mValue=mValue, + ) + # fmt: off @cute.jit def __call__( @@ -374,12 +396,14 @@ class FlashAttentionMLAForwardSm100: mPageTable: Optional[cute.Tensor] = None, window_size_left: Int32 | int | None = None, window_size_right: Int32 | int | None = None, - # Always keep stream as the last parameter (EnvStream: obtained implicitly via TVM FFI). stream: cuda.CUstream = None, + *, + mValue: Optional[cute.Tensor] = None, ): # fmt: on self.store_P = mP is not None self.store_row_max = mRowMax is not None + self.separate_value = mValue is not None if const_expr(self.has_qk): assert mQ is not None and mK is not None, "has_qk requires mQ and mK" @@ -402,11 +426,11 @@ class FlashAttentionMLAForwardSm100: *(cute.assume(s, divby=128 // mX.element_type.width) for s in mX.stride[:-1]), mX.stride[-1], ) - mQ, mQv, mK, mV, mO, mP = [ + mQ, mQv, mK, mV, mO, mP, mValue = [ cute.make_tensor(mX.iterator, cute.make_layout(mX.shape, stride=new_stride(mX))) if mX is not None else None - for mX in (mQ, mQv, mK, mV, mO, mP) + for mX in (mQ, mQv, mK, mV, mO, mP, mValue) ] # (b, s, h, d) -> (s, d, h, b) or @@ -420,16 +444,18 @@ class FlashAttentionMLAForwardSm100: else None for mX in (mQ, mQv, mO, mP) ] - mK, mV = [ + mK, mV, mValue = [ cute.make_tensor(mX.iterator, cute.select(mX.layout, mode=KV_layout_transpose)) if mX is not None else None - for mX in (mK, mV) + for mX in (mK, mV, mValue) ] # (s_k, dv, h_k, b) -> (dv, s_k, h_k, b) or # (total_k, dv, h_k) -> (dv, total_k, h_k) V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2] - mVt = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose)) + # MLA uses the same latent for scores and values; GQA supplies separate values. + value = mV if const_expr(mValue is None) else mValue + mVt = cute.make_tensor(value.iterator, cute.select(value.layout, mode=V_layout_transpose)) # (b, s_q, topk) -> (topk, s_q, b) or (total_q, topk) -> (topk, total_q) topk_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0] mIndexTopk = ( @@ -2605,7 +2631,9 @@ class FlashAttentionMLAForwardSm100: softmax = SoftmaxSm100.create( softmax_scale_log2, - rescale_threshold=8.0 if const_expr(self.dtype_Q.width == 16) else 0.0, + rescale_threshold=8.0 + if const_expr(self.dtype_Q.width == 16 and not self.separate_value) + else 0.0, softmax_scale=softmax_scale, ) softmax.reset() diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/sm100_hd256_2cta_fmha_forward.py b/python/sglang/kernels/ops/attention/flash_attn/cute/sm100_hd256_2cta_fmha_forward.py index f842a5cb7d..f5adbd6740 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/sm100_hd256_2cta_fmha_forward.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/sm100_hd256_2cta_fmha_forward.py @@ -341,9 +341,17 @@ class BlackwellFusedMultiHeadAttentionForward: stride_b_lse = h_r64 * h_k64 * s_lse64 if cum_seqlen_q is None else 0 # (s, d, ((h_r, h_k), b)) + if cutlass.const_expr(q_rank == 5): + q_row_stride = Int64(q_tensor.stride[1]) + q_head_stride = (Int64(q_tensor.stride[3]), Int64(q_tensor.stride[2])) + else: + q_row_stride = Int64(q_tensor.stride[q_rank - 3]) + head_stride = Int64(q_tensor.stride[q_rank - 2]) + q_head_stride = (head_stride, head_stride * h_r64) + q_batch_stride = Int64(q_tensor.stride[0]) if cum_seqlen_q is None else 0 q_layout = cute.make_layout( (s_q_total, d, ((h_r, h_k), b)), - stride=(d64 * h_r64 * h_k64, 1, ((d64, d64 * h_r64), stride_b_qo)), + stride=(q_row_stride, 1, (q_head_stride, q_batch_stride)), ) q = cute.make_tensor(q_tensor.iterator, q_layout) if cutlass.const_expr(mPageTable is not None): diff --git a/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py b/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py index f84a07eaa6..3648bd557c 100644 --- a/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py +++ b/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py @@ -12,6 +12,7 @@ from typing import Optional, Tuple import torch import triton import triton.language as tl +from triton.language.extra.cuda import libdevice def _pdl_supported() -> bool: @@ -60,6 +61,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel( HAS_GATE: tl.constexpr, MROPE: tl.constexpr, ENABLE_PDL: tl.constexpr, + WEIGHT_SHIFT: tl.constexpr = 1.0, + EXPLICIT_ROPE_FMA: tl.constexpr = False, ): token = tl.program_id(0) head = tl.program_id(1) @@ -86,7 +89,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel( w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) var = tl.sum(x * x, axis=0) / HEAD_DIM inv_rms = tl.rsqrt(var + EPS) - x_norm = (x * inv_rms * (w + 1.0)).to(out_dtype).to(tl.float32) + x_norm = (x * inv_rms * (w + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) # Pass-through tail [rotary_dim, head_dim) if HAS_PASS: @@ -104,8 +107,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel( wr2 = tl.load(w_ptr + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to( tl.float32 ) - xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(out_dtype).to(tl.float32) - xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(out_dtype).to(tl.float32) + xr1 = (xr1 * inv_rms * (wr1 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) + xr2 = (xr2 * inv_rms * (wr2 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32) if MROPE: axis = tl.load(mrope_axis_map_ptr + rot_offs, mask=rot_mask, other=0) @@ -121,8 +124,14 @@ def _fused_qk_rmsnorm_rope_gate_kernel( sin = tl.load( cos_sin_cache_ptr + cache_off + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0 ).to(tl.float32) - tl.store(out_base + rot_offs, (xr1 * cos - xr2 * sin), mask=rot_mask) - tl.store(out_base + HALF_ROTARY + rot_offs, (xr2 * cos + xr1 * sin), mask=rot_mask) + if EXPLICIT_ROPE_FMA: + first = tl.fma(xr1, cos, -libdevice.mul_rn(xr2, sin)) + second = tl.fma(xr2, cos, libdevice.mul_rn(xr1, sin)) + else: + first = xr1 * cos - xr2 * sin + second = xr2 * cos + xr1 * sin + tl.store(out_base + rot_offs, first, mask=rot_mask) + tl.store(out_base + HALF_ROTARY + rot_offs, second, mask=rot_mask) # Gate copy (Q heads only) if HAS_GATE and not is_k: diff --git a/python/sglang/kernels/ops/attention/gemma_qkv_norm_rope.py b/python/sglang/kernels/ops/attention/gemma_qkv_norm_rope.py new file mode 100644 index 0000000000..4009ea97fa --- /dev/null +++ b/python/sglang/kernels/ops/attention/gemma_qkv_norm_rope.py @@ -0,0 +1,124 @@ +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0. +"""In-place Gemma QKV normalization and partial NeoX RoPE.""" + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import ( + _fused_qk_rmsnorm_rope_gate_kernel, +) +from sglang.kernels.ops.layernorm.gemma4_fused_ops import _gemma_qkv_rmsnorm_store + + +@triton.jit +def _gemma_qkv_norm_rope( + Q, + K, + V, + QW, + KW, + CACHE, + POS, + SQ: tl.constexpr, + SK: tl.constexpr, + SV: tl.constexpr, + SC: tl.constexpr, + HQ: tl.constexpr, + HK: tl.constexpr, + D: tl.constexpr, + R: tl.constexpr, + EPS: tl.constexpr, + FP16: tl.constexpr, + BLOCK: tl.constexpr, + ROT_BLOCK: tl.constexpr, +): + head = tl.program_id(1) + if head < HQ + HK: + _fused_qk_rmsnorm_rope_gate_kernel( + Q, + K, + Q, + K, + Q, + QW, + KW, + CACHE, + POS, + POS, + SQ, + SK, + SQ, + SK, + SQ, + SC, + 1, + HQ, + HK, + D, + R, + R // 2, + BLOCK, + ROT_BLOCK, + EPS, + FP16, + R < D, + False, + False, + False, + WEIGHT_SHIFT=0.0, + EXPLICIT_ROPE_FMA=True, + ) + else: + cols = tl.arange(0, BLOCK) + _gemma_qkv_rmsnorm_store( + V, + QW, + SV, + tl.program_id(0), + head - HQ - HK, + cols, + cols < D, + D, + EPS, + False, + ) + + +def gemma_qkv_norm_rope( + q, + k, + v, + q_weight, + k_weight, + cache, + positions, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + eps, +): + """Normalize Q/K with weights and V without weights, then rotate Q/K.""" + _gemma_qkv_norm_rope[(q.shape[0], num_q_heads + 2 * num_kv_heads)]( + q, + k, + v, + q_weight, + k_weight, + cache, + positions, + q.stride(0), + k.stride(0), + v.stride(0), + cache.stride(0), + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + eps, + q.dtype == torch.float16, + triton.next_power_of_2(head_dim), + triton.next_power_of_2(rotary_dim // 2), + ) diff --git a/python/sglang/kernels/ops/kvcache/kv_indices.py b/python/sglang/kernels/ops/kvcache/kv_indices.py index 13aa9dc339..5779416ebd 100644 --- a/python/sglang/kernels/ops/kvcache/kv_indices.py +++ b/python/sglang/kernels/ops/kvcache/kv_indices.py @@ -33,6 +33,7 @@ def create_flashinfer_kv_indices_triton( req_to_token_ptr_stride, ENTRY_PAGE_SIZE: tl.constexpr = 1, TOKEN_BLOCK_PARALLEL: tl.constexpr = False, + token_mapping=None, ): """Gather per-request token ids into a flat CSR kv_indices stream. @@ -48,6 +49,7 @@ def create_flashinfer_kv_indices_triton( (which bottlenecks long-context spec decode, where this kernel runs every iteration). With the default, the kernel is the historical one-program-per-request loop and 1D launch sites are unaffected. + ``token_mapping`` remaps gathered token ids in the same pass, preserving -1. """ BLOCK_SIZE: tl.constexpr = 512 pid = tl.program_id(axis=0) @@ -95,6 +97,8 @@ def create_flashinfer_kv_indices_triton( mask=mask, ) data = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE + if token_mapping is not None: + data = tl.load(token_mapping + data, mask=mask & (data >= 0), other=-1) tl.store(kv_indices_ptr + kv_indices_offset + offset, data, mask=mask) diff --git a/python/sglang/kernels/ops/layernorm/__init__.py b/python/sglang/kernels/ops/layernorm/__init__.py index 79048da919..1e71b01c92 100644 --- a/python/sglang/kernels/ops/layernorm/__init__.py +++ b/python/sglang/kernels/ops/layernorm/__init__.py @@ -580,3 +580,18 @@ register_kernel( target="sglang.kernels.ops.layernorm.rms_normalize_hip:rms_normalize_triton", ) ) + +register_kernel( + KernelSpec( + op="layernorm.rmsnorm_fanout", + backend=KernelBackend.CUTE_DSL, + target="sglang.kernels.ops.layernorm.rmsnorm_fanout:rmsnorm_fanout", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} + ), + format_signature=FormatSignature( + supported_dtypes=_NORM_DTYPES, + description="Three weighted RMSNorm outputs sharing a single-CTA row reduction.", + ), + ) +) diff --git a/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py b/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py index a9e4eed6b7..988ffc4596 100644 --- a/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py +++ b/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py @@ -10,6 +10,8 @@ import torch import triton import triton.language as tl +from sglang.srt.utils import is_sm100_supported + @triton.jit def _gemma_rmsnorm_residual_kernel( @@ -277,7 +279,7 @@ def gemma_qkv_rmsnorm( assert k.stride(-1) == 1 and v.stride(-1) == 1 assert k_weight is not None and k_weight.shape[-1] == head_dim - if M <= 256: + if M <= 256 or (M == 512 and is_sm100_supported()): total_heads = num_q_heads + (2 * num_kv_heads if has_kv else 0) _gemma_qkv_rmsnorm_kernel[(M, total_heads)]( q, diff --git a/python/sglang/kernels/ops/layernorm/rmsnorm_fanout.py b/python/sglang/kernels/ops/layernorm/rmsnorm_fanout.py new file mode 100644 index 0000000000..161e49eb7f --- /dev/null +++ b/python/sglang/kernels/ops/layernorm/rmsnorm_fanout.py @@ -0,0 +1,211 @@ +# Copyright 2026 SGLang Team +# Copyright 2025 FlashInfer team +# Licensed under the Apache License, Version 2.0. +"""Three RMSNorm outputs sharing FlashInfer's input load and reduction. + +Uses the CuTe RMSNormKernel layout/reduction interface from FlashInfer 0.6.18. +The row-count dispatch covers the measured decode buckets on Blackwell. +""" + +from functools import cache + +import torch + +_FANOUT_MAX_ROWS = 2048 + + +def can_use_rmsnorm_fanout(x): + return ( + 0 < x.shape[0] <= _FANOUT_MAX_ROWS + and _fanout_kernel(x.dtype, x.shape[1]) is not None + ) + + +@cache +def _fanout_kernel(dtype, width): + if dtype not in (torch.bfloat16, torch.float16): + return None + kernel = _fanout_kernel_type()(_cutlass_dtype(dtype), width) + return kernel if kernel.cluster_n == 1 else None + + +def _cutlass_dtype(dtype): + import cutlass + + return cutlass.BFloat16 if dtype == torch.bfloat16 else cutlass.Float16 + + +def rmsnorm_fanout(x, weight0, weight1, weight2, eps): + """Normalize contiguous FP16/BF16 rows with three matching [H] weights.""" + outputs = tuple(torch.empty_like(x) for _ in range(3)) + compiled = _compile_fanout(x.device, x.dtype, x.shape[1]) + compiled(x, weight0, weight1, weight2, *outputs, x.shape[0], eps) + return outputs + + +@cache +def _compile_fanout(device, torch_dtype, width): + import cutlass + import cutlass.cute as cute + + dtype = _cutlass_dtype(torch_dtype) + kernel = _fanout_kernel(torch_dtype, width) + rows = cute.sym_int(64) + matrix = cute.runtime.make_fake_compact_tensor( + dtype, (rows, width), stride_order=(1, 0), assumed_align=16 + ) + weight = cute.runtime.make_fake_compact_tensor(dtype, (width,), assumed_align=16) + return cute.compile( + kernel, + matrix, + weight, + weight, + weight, + matrix, + matrix, + matrix, + cutlass.Int64(1), + cutlass.Float32(1e-6), + stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + + +@cache +def _fanout_kernel_type(): + import cutlass + import cutlass.cute as cute + from cutlass import Float32, Int64 + from flashinfer.norm.kernels.rmsnorm import RMSNormKernel + from flashinfer.norm.utils import predicate_k, row_reduce_sum_multirow + + class RMSNormFanout(RMSNormKernel): + @cute.jit + def __call__( + self, + x: cute.Tensor, + w0: cute.Tensor, + w1: cute.Tensor, + w2: cute.Tensor, + y0: cute.Tensor, + y1: cute.Tensor, + y2: cute.Tensor, + m: Int64, + eps: Float32, + stream, + ): + shape, stride = self._make_tv_layout( + self.threads_per_row, + self.rows_per_block, + self.vec_size, + self.num_vec_blocks, + ) + layout = cute.make_layout(shape, stride=stride) + tile = (self.rows_per_block, self.cols_per_tile) + self.kernel(x, w0, w1, w2, y0, y1, y2, m, eps, layout, tile).launch( + grid=[cute.ceil_div(m, self.rows_per_block), 1, 1], + block=[self.num_threads, 1, 1], + smem=self._smem_size_in_bytes(), + stream=stream, + ) + + @cute.kernel + def kernel( + self, + x: cute.Tensor, + w0: cute.Tensor, + w1: cute.Tensor, + w2: cute.Tensor, + y0: cute.Tensor, + y1: cute.Tensor, + y2: cute.Tensor, + m: Int64, + eps: Float32, + layout: cute.Layout, + tile: cute.Shape, + ): + tid, _, _ = cute.arch.thread_idx() + block, _, _ = cute.arch.block_idx() + smem = cutlass.utils.SmemAllocator() + if cutlass.const_expr(self.use_async_copy): + sx = smem.allocate_tensor( + x.element_type, + cute.make_ordered_layout(tile, order=(1, 0)), + byte_alignment=16, + ) + reduction = smem.allocate_tensor( + Float32, + cute.make_layout((self.rows_per_block, self.warps_per_row)), + byte_alignment=4, + ) + gx = cute.local_tile(x, tile, (block, 0)) + coords = cute.local_tile( + cute.make_identity_tensor(x.shape), tile, (block, 0) + ) + sync = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + x.element_type, + num_bits_per_copy=self.copy_bits, + ) + if cutlass.const_expr(self.use_async_copy): + async_copy = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp(), + x.element_type, + num_bits_per_copy=self.copy_bits, + ) + loader = cute.make_tiled_copy(async_copy, layout, tile) + else: + loader = cute.make_tiled_copy(sync, layout, tile) + copier = cute.make_tiled_copy(sync, layout, tile) + thread_x = loader.get_slice(tid) + thread_o = copier.get_slice(tid) + txg = thread_x.partition_S(gx) + txc = thread_x.partition_S(coords) + txr = cute.make_fragment_like(txg) + x_pred = predicate_k(txc, limit=self.H) + row_valid = txc[(0, 0), 0, 0][0] < m + if cutlass.const_expr(self.use_async_copy): + txs = thread_x.partition_D(sx) + if row_valid: + cute.copy(async_copy, txg, txs, pred=x_pred) + cute.arch.cp_async_commit_group() + cute.arch.cp_async_wait_group(0) + cute.autovec_copy(txs, txr) + else: + txr.store(cute.zeros_like(txr, dtype=x.element_type)) + if row_valid: + cute.copy(sync, txg, txr, pred=x_pred) + value = txr.load().to(Float32) + summed = row_reduce_sum_multirow( + value * value, self.threads_per_row, reduction, None, 1 + ) + rstd = cute.math.rsqrt(summed / Float32(self.H) + eps, fastmath=True) + cute.arch.barrier() + if cutlass.const_expr(self.use_async_copy): + cute.autovec_copy(txs, txr) + value = txr.load().to(Float32) + weights = (w0, w1, w2) + outputs = (y0, y1, y2) + wpred = predicate_k(thread_o.partition_S(coords), limit=self.H) + for index in cutlass.range_constexpr(3): + weight = weights[index] + expanded = cute.make_tensor( + weight.iterator, + cute.prepend( + weight.layout, cute.make_layout((tile[0],), stride=(0,)) + ), + ) + gw = cute.local_tile(expanded, tile, (0, 0)) + twg = thread_o.partition_S(gw) + twr = cute.make_fragment_like(twg) + cute.copy(sync, twg, twr, pred=wpred) + weight_value = thread_x.retile(twr).load().to(Float32) + gy = cute.local_tile(outputs[index], tile, (block, 0)) + tog = thread_o.partition_D(gy) + tor = cute.make_fragment_like(tog) + result = value * rstd * (weight_value + Float32(0.0)) + tor.store(result.to(outputs[index].element_type)) + if row_valid: + cute.copy(sync, tor, tog, pred=x_pred) + + return RMSNormFanout diff --git a/python/sglang/kernels/ops/sampling/__init__.py b/python/sglang/kernels/ops/sampling/__init__.py index c585956d27..f8ab5828a7 100644 --- a/python/sglang/kernels/ops/sampling/__init__.py +++ b/python/sglang/kernels/ops/sampling/__init__.py @@ -111,3 +111,17 @@ register_kernel( target="sglang.kernels.ops.sampling.murmur_hash:murmur_hash32", ) ) + + +register_kernel( + KernelSpec( + op="sampling.denoiser_statistics", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.sampling.denoiser_statistics:denoiser_statistics", + capabilities=frozenset({CapabilityRequirement.CUDA}), + format_signature=FormatSignature( + supported_dtypes=("float32",), + description="FP32 probabilities, entropy and raw-logit argmax for contiguous [B,M,V] logits; optional BF16 probabilities.", + ), + ) +) diff --git a/python/sglang/kernels/ops/sampling/denoiser_statistics.py b/python/sglang/kernels/ops/sampling/denoiser_statistics.py new file mode 100644 index 0000000000..e54979a9fd --- /dev/null +++ b/python/sglang/kernels/ops/sampling/denoiser_statistics.py @@ -0,0 +1,197 @@ +"""Chunked FP32 statistics for large-vocabulary block denoising. + +The partials carry a maximum, exponential sum and centered first moment. +Merging these computes entropy without materializing log-probabilities. A +separate pass writes FP32 probabilities for the existing categorical sampler +and self-conditioning GEMM. Sampling and its generator state stay unchanged. +""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _denoiser_partial_statistics( + LOGITS, + TEMPERATURES, + PARTIALS, + PARTIAL_ARGMAX, + VOCAB: tl.constexpr, + CANVAS: tl.constexpr, + ROWS: tl.constexpr, + CHUNKS: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + part = tl.program_id(1) + j = part * BLOCK + tl.arange(0, BLOCK) + temp = tl.load(TEMPERATURES + row // CANVAS) + raw = tl.load( + LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf") + ).to(tl.float32) + # Approximate reciprocal scaling changes sharp distributions enough to + # exceed the statistics tolerance. Match FP32 division rounding. + scaled = tl.div_rn(raw, temp) + mx = tl.max(scaled, 0) + d = scaled - tl.where(mx == -float("inf"), 0.0, mx) + e = tl.exp(d) + z = tl.sum(e, 0) + s = tl.sum(tl.where(j < VOCAB, d * e, 0.0), 0) + # Preserve PyTorch argmax's first-NaN and first-maximum behavior. + # An explicit NaN reduction is required; testing the reduced maximum + # itself is not equivalent after Triton lowering. + rawmax = tl.max(raw, 0) + has_nan = tl.sum(((j < VOCAB) & (raw != raw)).to(tl.int32), 0) > 0 + idx = tl.min( + tl.where( + (j < VOCAB) & tl.where(has_nan, raw != raw, raw == rawmax), j, 2147483647 + ), + 0, + ) + off = row * CHUNKS + part + tl.store(PARTIALS + off, mx) + tl.store(PARTIALS + ROWS * CHUNKS + off, z) + tl.store(PARTIALS + 2 * ROWS * CHUNKS + off, s) + tl.store(PARTIALS + 3 * ROWS * CHUNKS + off, rawmax) + tl.store(PARTIAL_ARGMAX + off, idx) + + +@triton.jit +def _denoiser_merge_statistics( + PARTIALS, + PARTIAL_ARGMAX, + NORMALIZER, + ENTROPY, + ARGMAX, + ROWS: tl.constexpr, + CHUNKS: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + p = tl.arange(0, BLOCK) + off = row * CHUNKS + p + mx = tl.load(PARTIALS + off, mask=p < CHUNKS, other=-float("inf")) + z = tl.load(PARTIALS + ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0) + moment = tl.load(PARTIALS + 2 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0) + maximum = tl.max(mx, 0) + shift = mx - maximum + alpha = tl.exp(shift) + denom = tl.sum(alpha * z, 0) + total = tl.sum( + tl.where(p < CHUNKS, alpha * (moment + tl.where(z > 0, z * shift, 0.0)), 0.0), 0 + ) + logz = tl.log(denom) + tl.store(NORMALIZER + row, maximum) + tl.store(NORMALIZER + ROWS + row, logz) + tl.store(ENTROPY + row, logz - total / denom) + rawmax = tl.load( + PARTIALS + 3 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=-float("inf") + ) + best = tl.max(rawmax, 0) + idx = tl.load(PARTIAL_ARGMAX + off, mask=p < CHUNKS, other=2147483647) + has_nan = tl.sum(((p < CHUNKS) & (rawmax != rawmax)).to(tl.int32), 0) > 0 + tl.store( + ARGMAX + row, + tl.min( + tl.where( + (p < CHUNKS) & tl.where(has_nan, rawmax != rawmax, rawmax == best), + idx, + 2147483647, + ), + 0, + ), + ) + + +@triton.jit +def _denoiser_write_probabilities( + LOGITS, + TEMPERATURES, + NORMALIZER, + PROBABILITIES, + SOFT_PROBABILITIES, + VOCAB: tl.constexpr, + CANVAS: tl.constexpr, + ROWS: tl.constexpr, + BLOCK: tl.constexpr, + WRITE_SOFT: tl.constexpr, +): + row = tl.program_id(0) + j = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + temp = tl.load(TEMPERATURES + row // CANVAS) + mx = tl.load(NORMALIZER + row) + logz = tl.load(NORMALIZER + ROWS + row) + raw = tl.load( + LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf") + ).to(tl.float32) + p = tl.exp(tl.div_rn(raw, temp) - mx - logz) + tl.store(PROBABILITIES + row.to(tl.int64) * VOCAB + j, p, mask=j < VOCAB) + if WRITE_SOFT: + tl.store( + SOFT_PROBABILITIES + row.to(tl.int64) * VOCAB + j, + p.to(tl.bfloat16), + mask=j < VOCAB, + ) + + +def denoiser_statistics( + logits: torch.Tensor, + temperatures: torch.Tensor, + soft_probabilities: torch.Tensor | None = None, +): + """Statistics for contiguous FP32 [B, M, V] logits and [B] temperatures. + + soft_probabilities, when supplied, is a contiguous BF16 buffer shaped like logits. + """ + assert logits.ndim == 3 and temperatures.shape == (logits.shape[0],) + block = 4096 + batch, canvas, vocab = logits.shape + rows = batch * canvas + parts = triton.cdiv(vocab, block) + partials = torch.empty((4, rows, parts), device=logits.device, dtype=torch.float32) + partial_argmax = torch.empty((rows, parts), device=logits.device, dtype=torch.int32) + normalizer = torch.empty((2, rows), device=logits.device, dtype=torch.float32) + probabilities = torch.empty_like(logits, dtype=torch.float32) + entropies = torch.empty((batch, canvas), device=logits.device, dtype=torch.float32) + argmax = torch.empty((batch, canvas), device=logits.device, dtype=torch.int64) + _denoiser_partial_statistics[(rows, parts)]( + logits, + temperatures, + partials, + partial_argmax, + vocab, + canvas, + rows, + parts, + block, + num_warps=4, + enable_fp_fusion=False, + ) + _denoiser_merge_statistics[(rows,)]( + partials, + partial_argmax, + normalizer, + entropies, + argmax, + rows, + parts, + triton.next_power_of_2(parts), + num_warps=4, + enable_fp_fusion=False, + ) + _denoiser_write_probabilities[(rows, triton.cdiv(vocab, block))]( + logits, + temperatures, + normalizer, + probabilities, + soft_probabilities if soft_probabilities is not None else probabilities, + vocab, + canvas, + rows, + block, + soft_probabilities is not None, + num_warps=4, + enable_fp_fusion=False, + ) + return probabilities, entropies, argmax diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index b4d51f651c..8fa7b5c2f2 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -6,7 +6,12 @@ The Triton kernels migrated here live in this package """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import KernelBackend, KernelSpec +from sglang.kernels.spec import ( + CapabilityRequirement, + FormatSignature, + KernelBackend, + KernelSpec, +) # (module, public_fn) migrated from speculative/triton_ops. _TRITON_KERNELS = [ @@ -37,4 +42,17 @@ for _mod, _fn in _TRITON_KERNELS: ) del _mod, _fn +register_kernel( + KernelSpec( + op="speculative.div_argmax", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.speculative.row_argmax:div_argmax", + capabilities=frozenset({CapabilityRequirement.CUDA}), + format_signature=FormatSignature( + supported_dtypes=("float32",), + description="Row-wise division and argmax on matching 2D tensors with unit column strides.", + ), + ) +) + __all__ = [] diff --git a/python/sglang/kernels/ops/speculative/row_argmax.py b/python/sglang/kernels/ops/speculative/row_argmax.py index 21adafc2e4..772cb25ff1 100644 --- a/python/sglang/kernels/ops/speculative/row_argmax.py +++ b/python/sglang/kernels/ops/speculative/row_argmax.py @@ -68,11 +68,16 @@ def _medium_argmax_partial_kernel( SX: tl.constexpr, SPLITS: tl.constexpr, BLOCK: tl.constexpr, + DIVISOR=None, + SD: tl.constexpr = 0, ): row, part = tl.program_id(0), tl.program_id(1) ix = part * BLOCK + tl.arange(0, BLOCK) valid = ix < N v = tl.load(X + row * SX + ix, valid, float("-inf")) + if DIVISOR is not None: + divisor = tl.load(DIVISOR + row * SD + ix, valid, 1.0) + v = tl.div_rn(v, divisor) i = tl.where(valid, ix, N) best_v, best_i = tl.reduce((v, i), 0, _argmax_pair) tl.store(PV + row * SPLITS + part, best_v) @@ -89,25 +94,50 @@ def _medium_argmax_final_kernel(PV, PI, OUT, SPLITS: tl.constexpr, BLOCK: tl.con tl.store(OUT + row, index.to(tl.int64)) +def div_argmax(x: torch.Tensor, divisor: torch.Tensor) -> torch.Tensor: + """FP32 row-wise ``(x / divisor).argmax(-1)`` with unit column strides.""" + assert x.ndim == 2 and x.dtype == torch.float32 and x.stride(1) == 1 + assert ( + divisor.shape == x.shape + and divisor.dtype == x.dtype + and divisor.device == x.device + and divisor.stride(1) == 1 + ) + return _medium_row_argmax(x, divisor) + + +def _medium_row_argmax(x, divisor=None): + rows, n = x.shape + block = 4096 if rows <= 256 else 8192 + splits = triton.cdiv(n, block) + values = torch.empty((rows, splits), dtype=torch.float32, device=x.device) + indices = torch.empty((rows, splits), dtype=torch.int32, device=x.device) + out = torch.empty(rows, dtype=torch.int64, device=x.device) + _medium_argmax_partial_kernel[(rows, splits)]( + x, + values, + indices, + n, + x.stride(0), + splits, + block, + DIVISOR=divisor, + SD=divisor.stride(0) if divisor is not None else 0, + num_warps=4, + ) + _medium_argmax_final_kernel[(rows,)]( + values, indices, out, splits, triton.next_power_of_2(splits), num_warps=1 + ) + return out + + def row_argmax(x: torch.Tensor) -> torch.Tensor: """``x.argmax(dim=-1)`` for FP32 speculative logits with a wide vocab.""" assert x.dim() == 2 and x.dtype == torch.float32 and x.stride(1) == 1 rows, n = x.shape - out = torch.empty((rows,), dtype=torch.int64, device=x.device) if rows > 64: - # Whole aligned tiles avoid the loop and unaligned partition starts. - # Larger batches use wider tiles to limit the total number of CTAs. - block = 4096 if rows <= 256 else 8192 - splits = triton.cdiv(n, block) - pv = torch.empty((rows, splits), dtype=torch.float32, device=x.device) - pi = torch.empty((rows, splits), dtype=torch.int32, device=x.device) - _medium_argmax_partial_kernel[(rows, splits)]( - x, pv, pi, n, x.stride(0), splits, block, num_warps=4 - ) - _medium_argmax_final_kernel[(rows,)]( - pv, pi, out, splits, triton.next_power_of_2(splits), num_warps=1 - ) - return out + return _medium_row_argmax(x) + out = torch.empty((rows,), dtype=torch.int64, device=x.device) pv = torch.empty((rows, _SPLITS), dtype=torch.float32, device=x.device) pi = torch.empty((rows, _SPLITS), dtype=torch.int32, device=x.device) _argmax_partial_kernel[(rows, _SPLITS)]( diff --git a/python/sglang/srt/arg_groups/dllm_hook.py b/python/sglang/srt/arg_groups/dllm_hook.py index 7dca439989..beae0a75a0 100644 --- a/python/sglang/srt/arg_groups/dllm_hook.py +++ b/python/sglang/srt/arg_groups/dllm_hook.py @@ -26,11 +26,10 @@ def handle_dllm_inference(server_args: Any): return from sglang.srt.dllm.algorithm import get_algorithm_cls + run_post_process_pass(server_args, _dllm_attention_backend) get_algorithm_cls(cfg.dllm_algorithm).configure_server_args(server_args) - # On AMD/HIP, disable cuda graph for DLLM (the attention_backend - # resolution moved to the pipeline: arg_groups/overrides.py - # _dllm_attention_backend, invoked below at its legacy slot). + # DLLM CUDA graphs are disabled on AMD/HIP. if get_platform().is_hip: if ( cfg.cuda_graph_config.decode.backend != Backend.DISABLED @@ -54,7 +53,6 @@ def handle_dllm_inference(server_args: Any): ), ) - run_post_process_pass(server_args, _dllm_attention_backend) run_post_process_pass(server_args, _dllm_overlap_disable) # The page-size alignment + block-size cap for dllm moved to the diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 04402f21ab..bb76413502 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1231,6 +1231,11 @@ def _fa4_page_constraint(view: Any) -> dict: # page_size==1, so skip the 128 auto-force for it and keep the default. and (view.speculative_eagle_topk or 0) <= 1 ): + if ( + "DiffusionGemmaForBlockDiffusion" + in model_config_of(view).hf_config.architectures + ): + return {"page_size": 1} logger.warning( f"FA4 backend only supports page size 128 for non-MLA model architectures, changing page_size from {view.page_size} to 128." ) @@ -1586,6 +1591,12 @@ def _dllm_attention_backend(view: Any) -> dict: from sglang.srt.dllm.algorithm import get_algorithm_cls algorithm_cls = get_algorithm_cls(view.dllm_algorithm) + if view.attention_backend in algorithm_cls.supported_attention_backends and all( + getattr(view, field, None) + in (None, *algorithm_cls.supported_attention_backends) + for field in ("prefill_attention_backend", "decode_attention_backend") + ): + return {} if backend := algorithm_cls.required_attention_backend: fields = ( "attention_backend", diff --git a/python/sglang/srt/dllm/algorithm/base.py b/python/sglang/srt/dllm/algorithm/base.py index 2c72e54dd1..47a221ed4a 100644 --- a/python/sglang/srt/dllm/algorithm/base.py +++ b/python/sglang/srt/dllm/algorithm/base.py @@ -34,6 +34,9 @@ class DllmAlgorithm: supported_architectures: Tuple[str, ...] = () requires_separate_context_encoding = False required_attention_backend: Optional[str] = None + reuse_forward_metadata: bool = False + supported_attention_backends: Tuple[str, ...] = () + capture_input_preparation: bool = False def __init__(self, config: DllmConfig): self.block_size = config.block_size @@ -113,11 +116,9 @@ class DllmAlgorithm: if all(start == self.block_size for start in start_list): return out.logits_output, [], None, None, out.can_run_graph - # NPU: attention metadata is stable across a block's denoise steps (the - # first forward above already planned it), so mark it ready once and let - # every later forward skip re-planning. - if _is_npu: - forward_batch.mark_forward_metadata_ready() + # The first forward has planned the fixed canvas's attention metadata. + if _is_npu or (self.reuse_forward_metadata and out.can_run_graph): + forward_batch.mark_forward_metadata_ready(replan_equivalent=not _is_npu) for _ in range(self.max_steps(self.block_size)): done = self.step(forward_batch, out.logits_output.full_logits, states) if all(done): diff --git a/python/sglang/srt/dllm/algorithm/gemma4_renoise.py b/python/sglang/srt/dllm/algorithm/gemma4_renoise.py index 038ae7d271..0ad0179ebe 100644 --- a/python/sglang/srt/dllm/algorithm/gemma4_renoise.py +++ b/python/sglang/srt/dllm/algorithm/gemma4_renoise.py @@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Any, List, Optional import torch -from sglang.srt.arg_groups.model_override_base import model_config_of +from sglang.srt.arg_groups.model_override_base import ( + attention_backends_of, + model_config_of, +) from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.dllm.algorithm.base import DllmAlgorithm, DllmRunOutput @@ -31,11 +34,39 @@ def _denoiser_statistics(logits: torch.Tensor, temperatures: torch.Tensor): _compiled_denoiser_statistics = torch.compile(_denoiser_statistics, dynamic=True) +def _use_split_denoiser_statistics(logits, temperatures): + return ( + logits.dtype == temperatures.dtype == torch.float32 + and logits.is_contiguous() + and temperatures.is_contiguous() + and logits.shape[-1] >= 65536 + and logits.shape[0] * logits.shape[1] >= 32 + and torch.cuda.get_device_capability(logits.device)[0] == 10 + ) + + +def _denoiser_statistics_cuda(logits, temperatures, *, write_soft=False): + soft_probabilities = None + if _use_split_denoiser_statistics(logits, temperatures): + from sglang.kernels.ops.sampling.denoiser_statistics import denoiser_statistics + + if write_soft: + soft_probabilities = torch.empty_like(logits, dtype=torch.bfloat16) + result = denoiser_statistics(logits, temperatures, soft_probabilities) + else: + result = _compiled_denoiser_statistics(logits, temperatures) + return *result, soft_probabilities + + def _sample_denoiser(probabilities: torch.Tensor, generator: torch.Generator): # The exponential-race formulation samples the same categorical distribution # as multinomial. Probabilities come from softmax, so multinomial's repeated # validation and device-to-host synchronization are unnecessary here. noise = torch.empty_like(probabilities).exponential_(1.0, generator=generator) + if probabilities.is_cuda: + from sglang.kernels.ops.speculative.row_argmax import div_argmax + + return div_argmax(probabilities, noise) return (probabilities / noise).argmax(dim=-1) @@ -43,6 +74,9 @@ class Gemma4Renoise(DllmAlgorithm): supported_architectures = ("DiffusionGemmaForBlockDiffusion",) requires_separate_context_encoding = True required_attention_backend = "triton" + reuse_forward_metadata = True + supported_attention_backends = ("triton", "fa4") + capture_input_preparation = True @classmethod def configure_server_args(cls, server_args: ServerArgs) -> None: @@ -66,16 +100,24 @@ class Gemma4Renoise(DllmAlgorithm): ): raise ValueError("DiffusionGemma requires tied word embeddings") + graph_config = cfg.cuda_graph_config + if graph_config.prefill.backend == Backend.FULL: + prefill_backend, _ = attention_backends_of(cfg) + if prefill_backend != "fa4": + raise ValueError( + "Gemma4Renoise FULL prefill CUDA graphs require prefill " + f"backend 'fa4'; got '{prefill_backend}'." + ) + else: + graph_config = with_phase( + graph_config, Phase.PREFILL, backend=Backend.DISABLED + ) declare_resolution( server_args, cls.__name__, disable_radix_cache=True, chunked_prefill_size=-1, - cuda_graph_config=with_phase( - cfg.cuda_graph_config, - Phase.PREFILL, - backend=Backend.DISABLED, - ), + cuda_graph_config=graph_config, ) @classmethod @@ -153,6 +195,11 @@ class Gemma4Renoise(DllmAlgorithm): self.seed = algorithm_config.get("seed") self.vocab_size = None self.embed_tokens = None + self._temperature_tables = {} + self._temperature_buffers = {} + self._zero_signals = {} + self.use_temperature_table = algorithm_config.get("use_temperature_table", True) + self.use_graph_input_preparation = config.capture_input_preparation if ( not isinstance(self.max_denoising_steps, int) @@ -226,40 +273,86 @@ class Gemma4Renoise(DllmAlgorithm): "argmax": current, "history": [], "self_conditioning": None, - "rng_state": generator.get_state(), + "generator": generator, "finished": False, } ) return states + @torch.no_grad() def prepare_inputs( self, model_runner: ModelRunner, forward_batch: ForwardBatch, states: List[Any], ) -> None: - self._write_input_ids(forward_batch, states) + if self.fdfo or all( + state["step"] == self.max_denoising_steps for state in states + ): + self._write_input_ids(forward_batch, states) signals = [state["self_conditioning"] for state in states] signal = next((value for value in signals if value is not None), None) - if signal is not None: - batched = torch.zeros( - (len(states), self.block_size, signal.shape[-1]), - dtype=signal.dtype, - device=signal.device, + if signal is not None or self.use_graph_input_preparation: + weight = self.embed_tokens.weight + key = (weight.device, weight.dtype, weight.shape[-1]) + if key not in self._zero_signals: + self._zero_signals[key] = weight.new_zeros( + self.block_size, weight.shape[-1] + ) + zero = self._zero_signals[key] + shape = (len(states) * self.block_size, weight.shape[-1]) + batched = forward_batch.dllm_input_preparation_state + if batched is None or batched.shape != shape: + batched = weight.new_empty(shape) + torch.stack( + [zero if value is None else value for value in signals], + out=batched.view(len(states), self.block_size, -1), ) - for index, value in enumerate(signals): - if value is not None: - batched[index].copy_(value) - signal = batched.view(len(states) * self.block_size, -1) + forward_batch.dllm_input_preparation_state = batched + signal = batched - forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds( - forward_batch.input_ids, signal + if self.use_graph_input_preparation: + forward_batch.input_embeds = None + else: + forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds( + forward_batch.input_ids, signal + ) + + @staticmethod + def prepare_graph_inputs(model, input_ids, state): + return model.prepare_dllm_input_embeds(input_ids, state) + + def _temperatures(self, logits, states): + if not self.use_temperature_table: + return logits.new_tensor( + [self._temperature(state["step"]) for state in states] + ) + key = (logits.device, logits.dtype) + if key not in self._temperature_tables: + self._temperature_tables[key] = logits.new_tensor( + [ + self._temperature(step) + for step in range(self.max_denoising_steps + 1) + ] + ) + table = self._temperature_tables[key] + if len(states) == 1: + step = states[0]["step"] + return table[step : step + 1] + buffer_key = (*key, len(states)) + if buffer_key not in self._temperature_buffers: + self._temperature_buffers[buffer_key] = logits.new_empty(len(states)) + return torch.stack( + [table[state["step"]] for state in states], + out=self._temperature_buffers[buffer_key], ) def _write_input_ids(self, forward_batch: ForwardBatch, states: List[Any]) -> None: - current = torch.stack([state["current"] for state in states]) - forward_batch.input_ids.copy_(current.view(-1)) + torch.stack( + [state["current"] for state in states], + out=forward_batch.input_ids.view(len(states), self.block_size), + ) def _soft_embeddings(self, probabilities: torch.Tensor) -> torch.Tensor: weight = self.embed_tokens.weight @@ -292,13 +385,23 @@ class Gemma4Renoise(DllmAlgorithm): self._write_input_ids(forward_batch, states) return [True] * len(states) - temperatures = logits.new_tensor( - [self._temperature(state["step"]) for state in states] - ) - statistics = ( - _compiled_denoiser_statistics if logits.is_cuda else _denoiser_statistics - ) - probabilities, token_entropies, argmax_tokens = statistics(logits, temperatures) + temperatures = self._temperatures(logits, states) + soft_probabilities = None + if logits.is_cuda: + probabilities, token_entropies, argmax_tokens, soft_probabilities = ( + _denoiser_statistics_cuda( + logits, + temperatures, + write_soft=( + self.embed_tokens.weight.dtype == torch.bfloat16 + and getattr(self.embed_tokens, "tp_size", 1) == 1 + ), + ) + ) + else: + probabilities, token_entropies, argmax_tokens = _denoiser_statistics( + logits, temperatures + ) sorted_entropy, indices = torch.sort(token_entropies, dim=-1) cumulative_entropy = torch.cumsum(sorted_entropy, dim=-1) sorted_selected = cumulative_entropy - sorted_entropy <= self.entropy_bound @@ -312,8 +415,7 @@ class Gemma4Renoise(DllmAlgorithm): if state["finished"]: continue active.append(index) - generator = torch.Generator(device=logits.device) - generator.set_state(state["rng_state"]) + generator = state["generator"] denoiser = _sample_denoiser(probabilities[index], generator) random_canvas = torch.randint( self.vocab_size, @@ -327,7 +429,8 @@ class Gemma4Renoise(DllmAlgorithm): if self.stability_threshold == 0: stable = torch.ones((), dtype=torch.bool, device=logits.device) elif len(history) == self.stability_threshold: - stable = (torch.stack(history) == argmax).all() + previous = history[0] if len(history) == 1 else torch.stack(history) + stable = (previous == argmax).all() else: stable = torch.zeros((), dtype=torch.bool, device=logits.device) history.append(argmax) @@ -335,7 +438,6 @@ class Gemma4Renoise(DllmAlgorithm): history.pop(0) state["step"] -= 1 state["argmax"] = argmax - state["rng_state"] = generator.get_state() finished.append((stable & confident[index]) | (state["step"] == 0)) # One device-to-host synchronization for the whole batch. In particular, @@ -348,7 +450,11 @@ class Gemma4Renoise(DllmAlgorithm): soft_embeds = None if not all(done): soft_embeds = self._soft_embeddings( - probabilities.reshape(-1, self.vocab_size) + ( + soft_probabilities + if soft_probabilities is not None + else probabilities + ).reshape(-1, self.vocab_size) ).view(len(states), self.block_size, -1) for index in active: state = states[index] diff --git a/python/sglang/srt/dllm/attention.py b/python/sglang/srt/dllm/attention.py new file mode 100644 index 0000000000..ad8e77b57e --- /dev/null +++ b/python/sglang/srt/dllm/attention.py @@ -0,0 +1,415 @@ +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0. +"""dLLM FA4 compute and graph workspaces over Triton paged-cache metadata.""" + +from dataclasses import dataclass +from functools import cache +from typing import ClassVar + +import torch + +from sglang.kernels.ops.attention.dllm_kv_pack import pack_prefix_current +from sglang.kernels.ops.attention.extend_attention import extend_attention_fwd_unified +from sglang.kernels.ops.attention.flash_attention_v4 import flash_attn_gqa_512 +from sglang.kernels.ops.attention.flash_attn.cute.interface import ( + flash_attn_varlen_func, +) +from sglang.srt.layers.attention.triton_backend import ( + ForwardMetadata, + update_sliding_window_buffer, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_capture_attention_variant, +) +from sglang.srt.utils import get_device_capability + + +class _DenseKVWorkspace: + def __init__(self, capacity, heads, dim, batch_size, device, dtype): + # TMA loads can include the masked tail beyond the packed sequences. + self.key = torch.zeros((capacity, heads, dim), device=device, dtype=dtype) + self.value = torch.zeros_like(self.key) + self.cu_seqlens = torch.empty(batch_size + 1, device=device, dtype=torch.int32) + self.indices = torch.arange(capacity, device=device, dtype=torch.int64) + self.window_start = torch.zeros(batch_size, device=device, dtype=torch.int32) + + def fits(self, capacity, batch_size, heads, dim): + return ( + capacity * heads * dim <= self.key.numel() + and capacity <= self.indices.numel() + and batch_size < self.cu_seqlens.numel() + ) + + def view(self, capacity, batch_size, heads, dim): + shape = (capacity, heads, dim) + elements = capacity * heads * dim + return ( + self.key.view(-1)[:elements].view(shape), + self.value.view(-1)[:elements].view(shape), + self.cu_seqlens[: batch_size + 1], + self.indices[:capacity], + self.window_start[:batch_size], + ) + + +DLLM_VARLEN = "dllm_varlen" +DLLM_FULL_WINDOW = "dllm_full_window" + + +@dataclass(frozen=True) +class DllmWindowGraphVariants: + window_size: int + block_size: int + capture_labels: ClassVar[tuple[str, ...]] = (DLLM_VARLEN, DLLM_FULL_WINDOW) + + def select(self, forward_batch: ForwardBatch, capture_batch_size: int) -> str: + lengths = forward_batch.seq_lens_cpu + if ( + forward_batch.batch_size > 0 + and capture_batch_size == forward_batch.batch_size + and forward_batch.forward_mode.is_dllm_extend() + and forward_batch.input_ids.numel() + == forward_batch.batch_size * self.block_size + and lengths is not None + and int(lengths[: forward_batch.batch_size].min()) - self.block_size + >= self.window_size + ): + return DLLM_FULL_WINDOW + return DLLM_VARLEN + + +class DllmFlashAttention: + """FA4 computation and context-encoding graphs for DiffusionGemma.""" + + def __init__(self, backend): + assert get_device_capability()[0] == 10, ( + "dLLM FA4 D256/D512 requires an SM100-family GPU." + ) + self.backend = backend + self.graph_mode = False + self._graph_workspaces = {} + self._eager_workspaces = {} + self._prefill_graph_metadata = {} + self.decode_graph_metadata = {} + self._graph_slots = self._graph_tokens = 0 + self._prefill_capture_sizes = set() + self._prefill_capture_max_requests = 0 + + def init_graph_state(self, max_requests, max_tokens): + self._graph_slots = max(self._graph_slots, max_requests) + self._graph_tokens = max(self._graph_tokens, max_tokens) + + def init_prefill_graph_state(self, max_requests, capture_tokens): + self.init_graph_state(max_requests, max(capture_tokens)) + self._prefill_capture_sizes = set(capture_tokens) + self._prefill_capture_max_requests = max_requests + + def graph_variants(self, block_size): + window = self.backend.sliding_window_size + if window is not None and window > 0: + return DllmWindowGraphVariants(window, block_size) + return None + + def _workspace(self, key, capacity, batch_size, heads, dim, query): + if self.graph_mode: + capacity = self._graph_slots * key[0] + self._graph_tokens + batch_size = self._graph_slots + captured = self._graph_workspaces.setdefault(key, []) + candidates = ( + captured + if self.graph_mode + else (w for group in self._graph_workspaces.values() for w in group) + ) + workspace = next( + ( + w + for w in candidates + if w.key.dtype == query.dtype + and w.fits(capacity, batch_size, heads, dim) + ), + None, + ) + if workspace is None: + workspace = self._eager_workspaces.get(key) + if workspace is None or not workspace.fits( + capacity, batch_size, heads, dim + ): + workspace = _DenseKVWorkspace( + capacity, heads, dim, batch_size, query.device, query.dtype + ) + if self.graph_mode: + # Keep earlier captures' allocations alive when a later runner needs more capacity. + captured.append(workspace) + self._eager_workspaces.pop(key, None) + else: + self._eager_workspaces[key] = workspace + return workspace + + def init_forward_metadata_out_graph(self, batch, in_capture): + self._eager_workspaces.clear() + self.graph_mode = True + if batch.forward_mode == ForwardMode.EXTEND: + return self.init_prefill_metadata(batch, in_capture) + bs = batch.batch_size + if not in_capture: + self.backend.forward_metadata = self.decode_graph_metadata[bs] + self.backend._init_forward_metadata_out_graph(batch, in_capture) + if in_capture: + self.decode_graph_metadata[bs] = self.backend.forward_metadata + + def init_prefill_metadata(self, batch, in_capture): + backend = self.backend + bs, tokens = batch.batch_size, batch.input_ids.numel() + key = (bs, tokens) + if in_capture: + self.init_graph_state(bs, tokens) + capacity = batch.max_seq_len_override or backend.max_context_len + swa = ( + backend.sliding_window_size is not None + and backend.sliding_window_size > 0 + ) + self._prefill_graph_metadata[key] = ForwardMetadata( + attn_logits=None, + attn_lse=None, + num_kv_splits=None, + max_extend_len=max(batch.extend_seq_lens_cpu), + kv_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device), + kv_indices=torch.empty( + bs * capacity, dtype=torch.int64, device=backend.device + ), + qo_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device), + custom_mask=None, + mask_indptr=None, + window_kv_indptr=torch.zeros( + bs + 1, dtype=torch.int32, device=backend.device + ) + if swa + else None, + window_kv_indices=torch.empty( + bs * backend.sliding_window_size, + dtype=torch.int64, + device=backend.device, + ) + if swa + else None, + window_num_kv_splits=None, + window_kv_offsets=torch.empty( + bs, dtype=torch.int64, device=backend.device + ) + if swa + else None, + swa_out_cache_loc=torch.empty_like(batch.out_cache_loc) + if backend.use_sliding_window_kv_pool + else None, + out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc) + if backend.kv_index_translator.is_translating + else None, + ) + metadata = self._prefill_graph_metadata[key] + backend._fill_kv_indptr_and_indices( + bs, + batch.extend_prefix_lens, + batch.req_pool_indices, + metadata.kv_indices, + kv_indptr=metadata.kv_indptr, + ) + metadata.qo_indptr[0].zero_() + torch.cumsum(batch.extend_seq_lens, dim=0, out=metadata.qo_indptr[1:]) + if metadata.window_kv_indices is not None: + _, _, _, offsets = update_sliding_window_buffer( + metadata.window_kv_indptr, + backend.kv_index_translator, + batch.req_pool_indices, + backend.sliding_window_size, + batch.extend_prefix_lens, + bs, + backend.device, + backend.token_to_kv_pool, + window_kv_indices=metadata.window_kv_indices, + ) + metadata.window_kv_offsets.copy_(offsets) + if metadata.swa_out_cache_loc is not None: + metadata.swa_out_cache_loc.copy_( + backend.kv_index_translator.sliding_window_write_loc_for( + batch.out_cache_loc + ) + ) + if metadata.out_cache_loc_full_physical is not None: + backend.kv_index_translator.fill_capture_write_loc( + out=metadata.out_cache_loc_full_physical, + forward_batch=batch, + width=metadata.out_cache_loc_full_physical.numel(), + ) + backend.forward_metadata = metadata + + def can_run_prefill_cuda_graph(self, batch): + query_limit = self.get_prefill_cuda_graph_max_query_len( + batch.input_ids.numel(), self._prefill_capture_max_requests + ) + return ( + batch.forward_mode == ForwardMode.EXTEND + and batch.input_ids.numel() in self._prefill_capture_sizes + and self.backend.dcp_size == 1 + and not batch.contains_image_inputs() + and ( + query_limit is None + or max(batch.extend_seq_lens_cpu, default=0) <= query_limit + ) + ) + + def get_prefill_cuda_graph_max_query_len(self, num_tokens, max_requests): + if num_tokens <= max_requests * self.backend.dllm_block_size: + return self.backend.dllm_block_size + return None + + @cache + def _supports_layer(self, layer, dtype): + pool = self.backend.token_to_kv_pool + buffers = ( + pool.get_key_buffer(layer.layer_id), + pool.get_value_buffer(layer.layer_id), + ) + return ( + layer.qk_head_dim == layer.v_head_dim + and layer.qk_head_dim in (256, 512) + and dtype in (torch.float16, torch.bfloat16) + and all( + buffer.ndim == 3 and buffer.dtype == dtype and buffer.stride(-1) == 1 + for buffer in buffers + ) + ) + + def forward_extend( + self, + layer, + q, + k, + v, + out, + kb, + vb, + qo, + ki, + ids, + mask, + causal, + mask_ptr, + max_q, + k_scale, + v_scale, + sm_scale=None, + **options, + ): + if ( + not self._supports_layer(layer, q.dtype) + or mask is not None + or options.get("sinks") is not None + or options.get("score_mod") is not None + or options.get("logit_cap", 0.0) + ): + return self.backend.extend_attention_fwd( + q, + k.contiguous(), + v.contiguous(), + out, + kb, + vb, + qo, + ki, + ids, + mask, + causal, + mask_ptr, + max_q, + k_scale, + v_scale, + sm_scale=sm_scale, + **options, + ) + bs, heads, dim = qo.numel() - 1, k.shape[1], k.shape[2] + is_window_layer = ( + layer.sliding_window_size is not None and layer.sliding_window_size >= 0 + ) + prefix_capacity = ( + self.backend.sliding_window_size + if is_window_layer + else self.backend.max_context_len + ) + prefix_tokens = bs * prefix_capacity if self.graph_mode else ids.numel() + capacity = prefix_tokens + q.shape[0] + key = (prefix_capacity, heads, dim, q.dtype) + workspace = self._workspace(key, capacity, bs, heads, dim, q) + dense_k, dense_v, cu_seqlens, dense_ids, window_start = workspace.view( + capacity, bs, heads, dim + ) + pack_prefix_current( + k, + v, + kb, + vb, + qo, + ki, + ids, + dense_k, + dense_v, + cu_seqlens, + prefix_capacity + max_q, + ) + window = options.get("sliding_window_size", -1) + if window > 0: + prefix_lens = ki[1:] - ki[:-1] + return extend_attention_fwd_unified( + q, + out, + dense_k, + dense_v, + k_scale, + v_scale, + qo, + cu_seqlens, + dense_ids, + prefix_lens, + max_q, + sm_scale=sm_scale, + is_causal=causal, + sliding_window_size=window, + window_start_pos=window_start, + page_size=1, + ) + if dim == 512: + return flash_attn_gqa_512( + q, + dense_k, + dense_v, + out, + cu_seqlens_q=qo, + cu_seqlens_k=cu_seqlens, + softmax_scale=sm_scale, + causal=causal, + ) + if ( + is_window_layer + and get_capture_attention_variant() == DLLM_FULL_WINDOW + and not causal + ): + return flash_attn_varlen_func( + q.view(bs, -1, *q.shape[1:]), + dense_k.view(bs, -1, heads, dim), + dense_v.view(bs, -1, heads, dim), + out=out.view(bs, -1, *out.shape[1:]), + softmax_scale=sm_scale, + causal=False, + ) + return flash_attn_varlen_func( + q, + dense_k, + dense_v, + out=out, + cu_seqlens_q=qo, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_q, + max_seqlen_k=prefix_capacity + max_q, + softmax_scale=sm_scale, + causal=causal, + ) diff --git a/python/sglang/srt/dllm/config.py b/python/sglang/srt/dllm/config.py index eeb6dba2fa..8dc81e8ce2 100644 --- a/python/sglang/srt/dllm/config.py +++ b/python/sglang/srt/dllm/config.py @@ -15,6 +15,7 @@ class DllmConfig: max_running_requests: int, first_done_first_out_mode: bool = False, requires_separate_context_encoding: bool = False, + capture_input_preparation: bool = False, ): self.algorithm = algorithm self.algorithm_config = algorithm_config @@ -23,6 +24,7 @@ class DllmConfig: self.max_running_requests = max_running_requests self.first_done_first_out_mode = first_done_first_out_mode self.requires_separate_context_encoding = requires_separate_context_encoding + self.capture_input_preparation = capture_input_preparation def validate_request(self, req) -> str | None: from sglang.srt.dllm.algorithm import get_algorithm_cls @@ -113,4 +115,8 @@ class DllmConfig: requires_separate_context_encoding=( algorithm_cls.requires_separate_context_encoding ), + capture_input_preparation=( + algorithm_cls.capture_input_preparation + and algorithm_config.get("capture_input_preparation", True) + ), ) diff --git a/python/sglang/srt/layers/attention/attention_registry.py b/python/sglang/srt/layers/attention/attention_registry.py index 873ddda494..6acd9ce7fd 100644 --- a/python/sglang/srt/layers/attention/attention_registry.py +++ b/python/sglang/srt/layers/attention/attention_registry.py @@ -253,6 +253,10 @@ def create_flashattention_v3_backend(runner): @register_attention_backend("fa4") def create_flashattention_v4_backend(runner): + if "DiffusionGemmaForBlockDiffusion" in runner.model_config.hf_config.architectures: + from sglang.srt.layers.attention.triton_backend import TritonAttnBackend + + return TritonAttnBackend(runner, dllm_fa4=True) from sglang.srt.layers.attention.flashattention_backend import ( FlashAttentionBackend, ) diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index 00da62095d..bcd7ec0f50 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -159,6 +159,7 @@ class AttentionBackend(ABC): # True when prefill graph metadata can use ForwardBatch.max_seq_len_override. supports_prefill_cuda_graph_max_context_size: bool = False + dllm_attention = None def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: """Declare where this backend's scheduler-shared reads end per mode. diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index 1ec7628b5a..3056235f6b 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -13,6 +13,7 @@ from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.runtime_context import ( + get_exec, get_spec, ) @@ -93,6 +94,10 @@ class HybridAttnBackend(AttentionBackend): def supports_full_cuda_graph_chunked_prefix(self) -> bool: return self.prefill_backend.supports_full_cuda_graph_chunked_prefix + @property + def dllm_attention(self): + return self.prefill_backend.dllm_attention + @property def supports_prefill_cuda_graph_max_context_size(self) -> bool: return self.prefill_backend.supports_prefill_cuda_graph_max_context_size @@ -125,12 +130,15 @@ class HybridAttnBackend(AttentionBackend): def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): self.decode_backend.init_cuda_graph_state(max_bs, max_num_tokens) - if get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill: - # When speculative decoding is enabled, we need to initialize the backend - # that will be used for target_verify. + if get_exec().dllm.dllm_algorithm is not None or ( + get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill + ): + # DLLM and prefill-backed verification capture the prefill backend. self.prefill_backend.init_cuda_graph_state(max_bs, max_num_tokens) def get_cuda_graph_seq_len_fill_value(self): + if get_exec().dllm.dllm_algorithm is not None: + return self.prefill_backend.get_cuda_graph_seq_len_fill_value() return self.decode_backend.get_cuda_graph_seq_len_fill_value() def init_mha_chunk_metadata( @@ -159,35 +167,12 @@ class HybridAttnBackend(AttentionBackend): ).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs) def validate_elastic_cuda_graph_recapture(self) -> None: - self.decode_backend.validate_elastic_cuda_graph_recapture() - - def forward( - self, - q: Optional[torch.Tensor] = None, # For full attention - k: Optional[torch.Tensor] = None, # For full attention - v: Optional[torch.Tensor] = None, # For full attention - layer: Optional[RadixAttention] = None, - forward_batch: Optional[ForwardBatch] = None, - save_kv_cache: bool = True, - *, - mixed_qkv: Optional[torch.Tensor] = None, # For linear attention - a: Optional[torch.Tensor] = None, # For linear attention - b: Optional[torch.Tensor] = None, # For linear attention - **kwargs, - ): - """Forward method that supports both regular attention (q, k, v) and linear attention (mixed_qkv, a, b).""" - backend = self._select_backend(forward_batch.forward_mode) - if mixed_qkv is not None: - return backend.forward( - layer=layer, - forward_batch=forward_batch, - save_kv_cache=save_kv_cache, - mixed_qkv=mixed_qkv, - a=a, - b=b, - **kwargs, - ) - return backend.forward(q, k, v, layer, forward_batch, save_kv_cache, **kwargs) + backend = ( + self.prefill_backend + if get_exec().dllm.dllm_algorithm is not None + else self.decode_backend + ) + backend.validate_elastic_cuda_graph_recapture() def forward_decode( self, diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py index d48ca563fb..e8e83fdf0f 100644 --- a/python/sglang/srt/layers/attention/tbo_backend.py +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -32,6 +32,10 @@ class TboAttnBackend(AttentionBackend): def supports_prefill_cuda_graph_max_context_size(self) -> bool: return self.primary.supports_prefill_cuda_graph_max_context_size + @property + def dllm_attention(self): + return self.primary.dllm_attention + @classmethod def init_new(cls, creator: Callable[[], AttentionBackend]): return cls( diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 5f58890921..e9b1fd99c6 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -154,6 +154,8 @@ class TritonAttnBackend(AttentionBackend): model_runner: ModelRunner, skip_prefill: bool = False, kv_indptr_buf: Optional[torch.Tensor] = None, + *, + dllm_fa4: bool = False, ): # Lazy import to avoid the initialization of cuda context from sglang.kernels.ops.attention.decode_attention import ( @@ -385,7 +387,9 @@ class TritonAttnBackend(AttentionBackend): if not self.skip_prefill: self.qo_indptr = torch.zeros( - (max_bs + 1,), dtype=torch.int64, device=model_runner.device + (max_bs + 1,), + dtype=torch.int32 if dllm_fa4 else torch.int64, + device=model_runner.device, ) self.mask_indptr = torch.zeros( @@ -409,6 +413,11 @@ class TritonAttnBackend(AttentionBackend): Lq=head_dim, Lv=head_dim ) self.extend_attention_block_m = block_m + if dllm_fa4: + from sglang.srt.dllm.attention import DllmFlashAttention + + self.dllm_attention = DllmFlashAttention(self) + self.supports_prefill_cuda_graph_max_context_size = True def get_num_kv_splits( self, @@ -498,8 +507,10 @@ class TritonAttnBackend(AttentionBackend): seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, kv_indices: torch.Tensor, + kv_indptr: Optional[torch.Tensor] = None, ) -> torch.Tensor: - kv_indptr = self.kv_indptr[: bs + 1] + if kv_indptr is None: + kv_indptr = self.kv_indptr[: bs + 1] kv_indptr[1:] = torch.cumsum(seq_lens, dim=0) self.kv_index_translator.fill_packed_read_stream( req_pool_indices=req_pool_indices[:bs], @@ -703,6 +714,13 @@ class TritonAttnBackend(AttentionBackend): forward_batch: ForwardBatch, in_capture: bool = False, ): + if self.dllm_attention is not None: + return self.dllm_attention.init_forward_metadata_out_graph( + forward_batch, in_capture + ) + self._init_forward_metadata_out_graph(forward_batch, in_capture) + + def _init_forward_metadata_out_graph(self, forward_batch, in_capture): bs = forward_batch.batch_size req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens @@ -810,6 +828,8 @@ class TritonAttnBackend(AttentionBackend): def init_forward_metadata(self, forward_batch: ForwardBatch): """Init auxiliary variables for triton attention backend.""" + if self.dllm_attention is not None: + self.dllm_attention.graph_mode = False self._dense_one_shot_kv_indptr = None bs = forward_batch.batch_size window_kv_indptr = self.window_kv_indptr @@ -1072,6 +1092,8 @@ class TritonAttnBackend(AttentionBackend): kv_indices_buf: Optional[torch.Tensor] = None, cuda_graph_num_kv_splits_buf: Optional[torch.Tensor] = None, ): + if self.dllm_attention is not None: + self.dllm_attention.init_graph_state(max_bs, max_num_tokens) self.cuda_graph_attn_logits = torch.zeros( (max_num_tokens, self.num_head, self.max_kv_splits, self.v_head_dim), dtype=torch.float32, @@ -1789,10 +1811,11 @@ class TritonAttnBackend(AttentionBackend): ): return o - self.extend_attention_fwd( + self._forward_extend_kernel( + layer, q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), - k.contiguous(), - v.contiguous(), + k.contiguous() if self.dllm_attention is None else k, + v.contiguous() if self.dllm_attention is None else v, o.view(-1, layer.tp_q_head_num, layer.v_head_dim), self.token_to_kv_pool.get_key_buffer(layer.layer_id), self.token_to_kv_pool.get_value_buffer(layer.layer_id), @@ -1818,6 +1841,11 @@ class TritonAttnBackend(AttentionBackend): ) return o + def _forward_extend_kernel(self, layer, *args, **kwargs): + if self.dllm_attention is not None: + return self.dllm_attention.forward_extend(layer, *args, **kwargs) + return self.extend_attention_fwd(*args, **kwargs) + def _dense_one_shot_kv_indptr_for(self, forward_batch: ForwardBatch): """Cumulative full sequence lengths addressing the one-shot K/V rows. @@ -2589,6 +2617,11 @@ def update_sliding_window_buffer( window_kv_indptr[-1], dtype=torch.int64, device=device ) window_kv_start_idx = seq_lens - window_kv_lens + token_mapping = ( + token_to_kv_pool.full_to_swa_index_mapping + if isinstance(token_to_kv_pool, SWAKVPool) + else None + ) translated = translator.fill_packed_read_stream( req_pool_indices=req_pool_indices[:bs], seq_lens=window_kv_lens, @@ -2597,6 +2630,7 @@ def update_sliding_window_buffer( out=window_kv_indices, kv_start_idx=window_kv_start_idx, sliding_window=translator.reads_are_translated, + token_mapping=token_mapping, ) if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool): kv_last_index = window_kv_indptr[-1] diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 0d33b54d34..f35efc4575 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -26,14 +26,14 @@ from torch import nn from sglang.kernels.ops.activation.softcap import ( softcap_inplace_logits as fused_softcap, ) +from sglang.kernels.ops.activation.softcap import ( + softcap_to_float32_logits, +) from sglang.srt.beam_search.logits_capture import BeamLogitsCapture from sglang.srt.distributed.device_communicators import triton_symm_mem_ag from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.aux_hidden_states import ( - AuxHiddenStates, - pack_aux_hidden_states, -) +from sglang.srt.layers.aux_hidden_states import AuxHiddenStates, pack_aux_hidden_states from sglang.srt.layers.dp_attention import ( DpPaddingMode, attn_tp_all_gather, @@ -945,11 +945,19 @@ class LogitsProcessor(nn.Module): "dp_logits_scatter_returned", logits_shape=tuple(logits.shape) ) + fuse_cast_softcap = ( + self.final_logit_softcapping + and logits.is_cuda + and logits.dtype in (torch.float16, torch.bfloat16) + ) logits = self._copy_logits_to_buffer( - logits, logits_metadata, use_buffer=use_logits_buffer + logits, + logits_metadata, + use_buffer=use_logits_buffer, + softcap=self.final_logit_softcapping if fuse_cast_softcap else None, ) - if self.final_logit_softcapping: + if self.final_logit_softcapping and not fuse_cast_softcap: if not (_is_npu or _is_cpu): fused_softcap(logits, self.final_logit_softcapping) else: @@ -1148,11 +1156,19 @@ class LogitsProcessor(nn.Module): logits: torch.Tensor, logits_metadata: LogitsMetadata, use_buffer: bool = True, + softcap: Optional[float] = None, ) -> torch.Tensor: logits_buffer = logits_metadata.next_token_logits_buffer if use_buffer else None if logits.shape[-1] > self.vocab_size: logits = logits[:, : self.vocab_size] logits_width = logits.shape[-1] + if softcap is not None: + if logits_buffer is None or tuple(logits_buffer.shape) != tuple( + logits.shape + ): + logits_buffer = torch.empty_like(logits, dtype=torch.float32) + assert logits_buffer.dtype == torch.float + return softcap_to_float32_logits(logits, softcap, logits_buffer) # The shared logits buffer is keyed by vocab width and rows; skip it # when this batch has a different logits shape than the graph buffer. if logits_buffer is not None and tuple(logits_buffer.shape) == tuple( diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC.json new file mode 100644 index 0000000000..22c61cdeee --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC.json @@ -0,0 +1,163 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1 + }, + "129": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "255": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "257": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "510": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "511": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "513": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "514": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "1022": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "1023": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "1025": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "1026": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "2046": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "2047": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2049": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2050": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + } +} diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC_down.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC_down.json new file mode 100644 index 0000000000..edb97b166e --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=128,N=704,device_name=NVIDIA_B300_SXM6_AC_down.json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1 + }, + "129": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "255": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "257": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "510": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "511": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "513": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "514": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "1022": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "1023": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "1025": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": false + }, + "1026": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "2046": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "2047": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2049": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "USE_TMA": true + }, + "2050": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8 + } +} diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index ebfe6dd003..c4461e2d7b 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -194,11 +194,12 @@ class KVIndexTranslator: out: torch.Tensor, kv_start_idx: Optional[torch.Tensor] = None, sliding_window: bool = False, + token_mapping: Optional[torch.Tensor] = None, ) -> bool: """Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and report whether they came out translated. - Non-unified: the historical gather straight from ``req_to_token``. + Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``. Unified: one fused gather-and-translate, so no caller needs a ``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per resident token, a length the pool bounds. @@ -206,8 +207,8 @@ class KVIndexTranslator: ``sliding_window`` selects the swa sub-pool's own id space, built from VIRTUAL ids and never chained through full-physical. A ``False`` return means the ids are still VIRTUAL: the DCP path defers translation to - ``translate_dcp_read_ids``, and a static SWA pool maps the full ids - through its own full->swa table. + ``translate_dcp_read_ids``. A static SWA pool can pass its full->swa + table as ``token_mapping`` to fuse that translation into the gather. """ # `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices # (the padded graph buffer), and the extra lanes have no length to bound. @@ -228,8 +229,9 @@ class KVIndexTranslator: out, self.req_to_token.stride(0), ENTRY_PAGE_SIZE=1, + token_mapping=token_mapping, ) - return False + return token_mapping is not None if sliding_window: assert self._swa_v2p_table is not None, ( diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 8009e0a9b9..fd783f7dc5 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -556,6 +556,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # For input embeddings input_embeds: Optional[torch.Tensor] = None + dllm_input_preparation_state: Optional[torch.Tensor] = None # For token embedding overrides (sparse replacement at specific positions) replace_embeds: Optional[torch.Tensor] = None replace_positions: Optional[torch.Tensor] = None diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 2d6240f9cb..67cb669058 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -39,9 +39,7 @@ from torch.profiler import ProfilerActivity, profile from sglang.srt.compilation import torch_compile_decoration from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config -from sglang.srt.distributed.parallel_state import ( - graph_capture, -) +from sglang.srt.distributed.parallel_state import graph_capture from sglang.srt.dllm.config import DllmConfig from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import ( @@ -63,6 +61,8 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.model_executor.cuda_graph_buffer_registry import ( CudaGraphBufferRegistry, + GraphSlot, + PaddingPolicy, build_decode_registry, ) from sglang.srt.model_executor.forward_batch_info import ( @@ -89,12 +89,8 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor BreakableCudaGraphBackend, ) from sglang.srt.model_executor.runner_backend.utils import resolve_decode_backend -from sglang.srt.model_executor.runner_backend_utils import ( - CUDA_GRAPH_CAPTURE_FAILED_MSG, -) -from sglang.srt.model_executor.runner_utils.buffers import ( - DecodeInputBuffers, -) +from sglang.srt.model_executor.runner_backend_utils import CUDA_GRAPH_CAPTURE_FAILED_MSG +from sglang.srt.model_executor.runner_utils.buffers import DecodeInputBuffers from sglang.srt.model_executor.runner_utils.capture_mode import ( _set_capture_attention_variant, _set_capture_lora_variant, @@ -108,12 +104,7 @@ from sglang.srt.model_executor.runner_utils.pool import ( ) from sglang.srt.model_executor.runner_utils.shared_read_event import make_external_event from sglang.srt.multiplex.pdmux_context import get_current_stream_idx, get_stream_groups -from sglang.srt.runtime_context import ( - get_exec, - get_flags, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_exec, get_flags, get_parallel, get_spec from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout from sglang.srt.utils import ( empty_context, @@ -230,6 +221,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): pluggable self.backend that handles the actual capture/replay. """ + dllm_input_preparation = None + dllm_attention = None + def __init__( self, model_runner: ModelRunner, @@ -278,6 +272,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.dllm_config = DllmConfig.from_server_args(model_runner.server_args) self.is_dllm = self.dllm_config is not None + self.dllm_input_preparation = None + if self.is_dllm and self.dllm_config.capture_input_preparation: + from sglang.srt.dllm.algorithm import get_algorithm_cls + + self.dllm_input_preparation = get_algorithm_cls( + self.dllm_config.algorithm + ).prepare_graph_inputs self.dllm_uses_input_embeds = ( self.is_dllm and self.dllm_config.requires_separate_context_encoding ) @@ -309,13 +310,22 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): elif self.is_dllm: self.capture_forward_mode = ForwardMode.DLLM_EXTEND + self.dllm_attention = self.attn_backend.dllm_attention self.attention_graph_variants: Optional[AttentionGraphVariants] = ( - create_attention_graph_variants(model_runner.model_config.hf_config) + self.dllm_attention.graph_variants(self.captured_req_width) + if self.dllm_attention is not None and self.is_dllm + else create_attention_graph_variants(model_runner.model_config.hf_config) or create_dsv41_candidate_graph_variants( model_runner, self.capture_forward_mode, self.captured_req_width ) ) + if ( + self.dllm_attention is not None + and self.attention_graph_variants is not None + ): + self._resolve_attention_variant = self.attention_graph_variants.select + # --- bucket sizes --------------------------------------------- self.capture_bs, self.compile_bs = get_batch_sizes_to_capture( model_runner, self.captured_req_width @@ -461,6 +471,17 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): num_dp_ranks=self.num_dp_ranks, source=self.buffers, ) + if self.dllm_input_preparation is not None: + width = self.buffers.input_embeds.shape[-1] + self.buffer_registry.register_slot( + GraphSlot( + "dllm_input_preparation_state", + lambda bs, tokens: (tokens, width), + self.buffers.input_embeds.dtype, + axis="tokens", + padding_policy=PaddingPolicy.ZERO, + ) + ) # Captures the per-replay attention-metadata prep into a small CUDA # graph; see metadata_glue_graph.py for the correctness contract. @@ -589,9 +610,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): return [num_tokens] return None - def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]: + def _resolve_attention_variant( + self, forward_batch: ForwardBatch, capture_batch_size: int + ) -> Optional[str]: variants = self.attention_graph_variants - return variants.select(forward_batch) if variants is not None else None + if variants is None: + return None + return variants.select(forward_batch) def _resolve_lora_variant(self, forward_batch: ForwardBatch): if not self.record_nolora_graph: @@ -705,7 +730,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): stream_idx=get_current_stream_idx() if self.enable_pdmux else None, variant_label=self._resolve_lora_variant(forward_batch), attention_variant=( - self._resolve_attention_variant(forward_batch) + self._resolve_attention_variant(forward_batch, cuda_graph_bs) if self.disable_padding else None ), @@ -1114,10 +1139,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): else [(None, None)] ) variants = self.attention_graph_variants - attention_variants = ( - variants.capture_labels if variants is not None else (None,) - ) for bs in capture_range: + attention_variants = ( + variants.capture_labels if variants is not None else (None,) + ) if get_parallel().tp_rank == 0: avail_mem = get_available_gpu_memory( self.model_runner.device, @@ -1220,7 +1245,15 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): and "input_embeds" in inspect.signature(forward).parameters and not hasattr(self.model_runner.model, "forward_embed") ): - kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens] + if self.dllm_input_preparation is None: + kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens] + else: + state = self.buffer_registry.get_slot( + "dllm_input_preparation_state" + ).buffer[:num_tokens] + kwargs["input_embeds"] = self.dllm_input_preparation( + self.model_runner.model, forward_batch.input_ids, state + ) out = forward( forward_batch.input_ids, @@ -1280,7 +1313,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): forward_batch: ForwardBatch, pp_proxy_tensors: Optional[PPProxyTensors] = None, ): - if self.dllm_uses_input_embeds and forward_batch.input_embeds is None: + if ( + self.dllm_uses_input_embeds + and self.dllm_input_preparation is None + and forward_batch.input_embeds is None + ): raise ValueError( "Diffusion graph replay requires prepared input embeddings" ) @@ -1294,6 +1331,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.deepep_adapter.replay() if not forward_batch.needs_forward_metadata_init(): + if self.dllm_input_preparation is not None: + slot = self.buffer_registry.get_slot("dllm_input_preparation_state") + source = forward_batch.dllm_input_preparation_state + padded_tokens = self.bs * self.captured_req_width + slot.buffer[: self.raw_num_token].copy_(source) + slot.reset_padding(self.raw_num_token, padded_tokens) + self.buffers.input_ids[self.raw_num_token : padded_tokens].zero_() # Pre-planned (plan-stream load_batch already ran). # In speculative decoding, these two fields are still needed. graph_size_key = ( @@ -1339,7 +1383,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): forward_batch.input_embeds ) variant_label = self._resolve_lora_variant(forward_batch) - attention_variant = self._resolve_attention_variant(forward_batch) + attention_variant = self._resolve_attention_variant(forward_batch, self.bs) stream_idx = get_current_stream_idx() if self.enable_pdmux else None self._replay_graph_key = self._make_graph_key( graph_size_key, stream_idx, variant_label, attention_variant @@ -1472,7 +1516,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.model_runner.hisparse_coordinator.num_real_reqs.fill_(raw_bs) variant_label = self._resolve_lora_variant(forward_batch) - attention_variant = self._resolve_attention_variant(forward_batch) + attention_variant = self._resolve_attention_variant(forward_batch, bs) stream_idx = get_current_stream_idx() if self.enable_pdmux else None self._replay_graph_key = self._make_graph_key( graph_size_key, stream_idx, variant_label, attention_variant diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 6edd052207..674a1b44fb 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -288,6 +288,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): _use_draft_input_embeds = False _backend_can_run_prefill_cuda_graph = None + dllm_attention = None def __init__(self, model_runner: ModelRunner): if get_schedule().enable_mixed_chunk: @@ -297,8 +298,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): f"graph backend; got '{backend}'." ) super().__init__(model_runner) - self._backend_can_run_prefill_cuda_graph = getattr( - model_runner.attn_backend, "can_run_prefill_cuda_graph", None + self.dllm_attention = model_runner.attn_backend.dllm_attention + self._backend_can_run_prefill_cuda_graph = ( + self.dllm_attention.can_run_prefill_cuda_graph + if self.dllm_attention is not None + else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None) ) # --- model flags ---------------------------------------------- self.quant_config = getattr(model_runner.model, "quant_config", None) @@ -481,6 +485,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): max_req = prefill_config.full_prefill_max_req assert max_req is not None, "full_prefill_max_req must be resolved" self._capture_req_slots = max_req + if self.dllm_attention is not None: + self.dllm_attention.init_prefill_graph_state( + max_req, self.capture_num_tokens + ) # BCG/Full record LoRA kernels, so the metadata they read must live in # static buffers refreshed in place per batch; unsupported LoRA @@ -505,7 +513,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # This flag controls whether the model dispatches through the distinct # chunked-prefix topology; backend capability is validated separately. self._capture_chunked_prefix = ( - self._is_full_backend and not get_schedule().disable_chunked_prefix_cache + self._is_full_backend + and not get_schedule().disable_chunked_prefix_cache + and self.dllm_attention is None ) self._prefix_chunk_len = 0 self._prefix_chunk_capacity = 0 @@ -1406,6 +1416,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): context_length = min( self.max_context_size or model_context_length, model_context_length ) + if self.dllm_attention is not None and self._is_full_backend: + query_limit = self.dllm_attention.get_prefill_cuda_graph_max_query_len( + num_tokens, self._capture_req_slots + ) + if query_limit is not None: + context_length = min(context_length, query_limit) # A prefill bucket is an aggregate token count. Capture it as the # fewest synthetic requests, with every request containing no more # than context_length tokens. @@ -2153,7 +2169,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): def _finalize_execute_output( self, output - ) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]: + ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: + if output is None: + return None if isinstance(output, LogitsProcessorOutput): return self._trim_logits_output(output) if isinstance(output, EmbeddingPoolerOutput): @@ -2170,7 +2188,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): def execute( self, forward_batch: ForwardBatch, **kwargs - ) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]: + ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: self._validate_capture_hidden_mode(forward_batch) with self.backend.replay_session(): static_forward_batch = self.load_batch(forward_batch, **kwargs) diff --git a/python/sglang/srt/models/gemma4_diffusion.py b/python/sglang/srt/models/gemma4_diffusion.py index c6a01f48a0..503d12d605 100644 --- a/python/sglang/srt/models/gemma4_diffusion.py +++ b/python/sglang/srt/models/gemma4_diffusion.py @@ -21,7 +21,13 @@ import torch from torch import nn from transformers import PreTrainedModel +from sglang.kernels.ops.attention.gemma_qkv_norm_rope import gemma_qkv_norm_rope from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm +from sglang.kernels.ops.layernorm.rmsnorm_fanout import ( + can_use_rmsnorm_fanout, + rmsnorm_fanout, +) +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.layers.activation import GeluAndMul from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm from sglang.srt.layers.linear import ( @@ -57,7 +63,7 @@ from sglang.srt.models.gemma4_mm import ( ) from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import add_prefix, make_pp_layers +from sglang.srt.utils import add_prefix, is_sm100_supported, make_pp_layers logger = logging.getLogger(__name__) @@ -162,7 +168,28 @@ class DiffusionGemmaAttention(nn.Module): qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - if q.is_cuda: + fuse_rope = ( + q.is_cuda + and q.dtype in (torch.bfloat16, torch.float16) + and 0 < q.shape[0] <= 1024 + and is_sm100_supported() + ) + if fuse_rope: + gemma_qkv_norm_rope( + q, + k, + v, + self.q_norm.weight, + self.k_norm.weight, + self.rotary_emb.cos_sin_cache, + positions, + self.num_heads, + self.num_kv_heads, + self.head_dim, + self.rotary_emb.rotary_dim, + self.q_norm.eps, + ) + elif q.is_cuda: gemma_qkv_rmsnorm( q, k, @@ -185,7 +212,8 @@ class DiffusionGemmaAttention(nn.Module): v.unflatten(-1, (self.num_kv_heads, self.head_dim)) ).flatten(-2, -1) - q, k = self.rotary_emb(positions, q, k) + if not fuse_rope: + q, k = self.rotary_emb(positions, q, k) attn = ( self.attn @@ -289,6 +317,7 @@ class DiffusionGemmaDecoderLayer(nn.Module): ), ) + self.use_fanout = is_sm100_supported() and not is_batch_invariant_mode_enabled() eps = config.rms_norm_eps self.input_layernorm = RMSNorm(config.hidden_size, eps=eps) self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=eps) @@ -310,14 +339,30 @@ class DiffusionGemmaDecoderLayer(nn.Module): hidden_states = residual + hidden_states residual = hidden_states - h1 = self.post_feedforward_layernorm_1( - self.mlp(self.pre_feedforward_layernorm(residual)) - ) + use_fanout = self.use_fanout and can_use_rmsnorm_fanout(residual) + if use_fanout: + if not self.router._scale_fused: + self.router.fuse_scale() + dense_in, moe_in, router_in = rmsnorm_fanout( + residual, + self.pre_feedforward_layernorm.weight, + self.pre_feedforward_layernorm_2.weight, + self.router.norm.weight, + self.pre_feedforward_layernorm.variance_epsilon, + ) + else: + dense_in = self.pre_feedforward_layernorm(residual) + h1 = self.post_feedforward_layernorm_1(self.mlp(dense_in)) + del dense_in + if use_fanout: + router_logits, _ = self.router.proj(router_in) + del router_in + else: + moe_in = self.pre_feedforward_layernorm_2(residual) + router_logits = self.router(residual) # MoE branch: the router gates on the RAW residual (the router's input # norm is with_scale=False, so ln2's learned per-channel weights would # change the routed direction). Experts still get the ln2-normed input. - moe_in = self.pre_feedforward_layernorm_2(residual) - router_logits = self.router(residual) h2 = self.post_feedforward_layernorm_2(self.moe(moe_in, router_logits)) hidden_states = self.post_feedforward_layernorm(h1 + h2) @@ -479,6 +524,10 @@ class DiffusionGemmaForBlockDiffusion(PreTrainedModel): **kwargs, ): is_context_encoding = not forward_batch.forward_mode.is_dllm_extend() + if not is_context_encoding and input_embeds is None: + input_embeds = self.prepare_dllm_input_embeds( + input_ids, forward_batch.dllm_input_preparation_state + ) if is_context_encoding and forward_batch.contains_image_inputs(): Gemma4ForConditionalGeneration.prepare_attn_masks( self, forward_batch, input_ids, torch.bool diff --git a/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py b/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py index 2ab6f86022..8b663051d0 100644 --- a/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py +++ b/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py @@ -5,8 +5,8 @@ import unittest import torch from sglang.srt.dllm.algorithm.gemma4_renoise import ( - _compiled_denoiser_statistics, _denoiser_statistics, + _denoiser_statistics_cuda, _sample_denoiser, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -17,16 +17,56 @@ register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-l @unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") class TestGemma4SamplingCUDA(CustomTestCase): + def test_statistics_bf16_output_and_changed_input_graph_replay(self): + if torch.cuda.get_device_capability()[0] != 10: + self.skipTest("Split statistics dispatch requires Blackwell") + logits = torch.zeros(2, 33, 65537, device="cuda") + temperatures = torch.tensor([0.4, 0.8], device="cuda") + _denoiser_statistics_cuda(logits, temperatures, write_soft=True) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = _denoiser_statistics_cuda(logits, temperatures, write_soft=True) + for kind in ( + "uniform", + "ties", + "masked_chunk", + "all_masked", + "infinity", + "nan", + ): + with self.subTest(kind=kind): + logits.zero_() + temperatures.copy_(temperatures.flip(0)) + if kind == "ties": + logits[:, :, 0] = logits[:, :, -1] = 5 + elif kind == "masked_chunk": + logits[:, :, :8192] = -float("inf") + elif kind == "all_masked": + logits.fill_(-float("inf")) + elif kind == "infinity": + logits[:, :, 123] = float("inf") + elif kind == "nan": + logits[:, :, 123] = logits[:, :, 45000] = float("nan") + graph.replay() + expected = _denoiser_statistics(logits, temperatures) + for got, want in zip(actual, expected): + torch.testing.assert_close( + got, want, rtol=2e-5, atol=2e-6, equal_nan=True + ) + torch.testing.assert_close( + actual[3], actual[0].bfloat16(), rtol=0, atol=0, equal_nan=True + ) + def test_full_vocabulary_statistics_and_sampling(self): generator = torch.Generator(device="cuda").manual_seed(123) for batch_size in (1, 3): with self.subTest(batch_size=batch_size): logits = torch.randn( - batch_size, 16, 262144, device="cuda", generator=generator + batch_size, 32, 262144, device="cuda", generator=generator ) temperatures = torch.linspace(0.4, 0.8, batch_size, device="cuda") expected = _denoiser_statistics(logits, temperatures) - actual = _compiled_denoiser_statistics(logits, temperatures) + actual = _denoiser_statistics_cuda(logits, temperatures) for got, want in zip(actual, expected): torch.testing.assert_close(got, want, rtol=2e-5, atol=2e-6) a = torch.Generator(device="cuda").manual_seed(42) diff --git a/test/registered/unit/dllm/test_gemma4_cuda_graph.py b/test/registered/unit/dllm/test_gemma4_cuda_graph.py index d082361b0a..418f4de935 100644 --- a/test/registered/unit/dllm/test_gemma4_cuda_graph.py +++ b/test/registered/unit/dllm/test_gemma4_cuda_graph.py @@ -35,6 +35,7 @@ class _ReadStream: out, kv_start_idx=None, sliding_window=False, + token_mapping=None, ): for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)): start = 0 if kv_start_idx is None else int(kv_start_idx[i]) diff --git a/test/registered/unit/dllm/test_gemma4_diffusion.py b/test/registered/unit/dllm/test_gemma4_diffusion.py index 4dda30a620..1012ae5ca5 100644 --- a/test/registered/unit/dllm/test_gemma4_diffusion.py +++ b/test/registered/unit/dllm/test_gemma4_diffusion.py @@ -71,6 +71,7 @@ def _diffusion_model_stub(): def _dispatch_batch(*, encoder): return SimpleNamespace( forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND), + dllm_input_preparation_state=None, contains_image_inputs=lambda: True, ) diff --git a/test/registered/unit/dllm/test_gemma4_renoise.py b/test/registered/unit/dllm/test_gemma4_renoise.py index 0b91b9f679..12335de663 100644 --- a/test/registered/unit/dllm/test_gemma4_renoise.py +++ b/test/registered/unit/dllm/test_gemma4_renoise.py @@ -45,6 +45,7 @@ def _batch(rids, block_size, *, sampling_seeds=None, encoder=False, empty=False) sampling_info=sampling_info, forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND), input_embeds=None, + dllm_input_preparation_state=None, ) @@ -203,7 +204,7 @@ class TestGemma4Renoise(unittest.TestCase): dcp_size=1, attn_cp_size=1, attention_backend="flashinfer", - prefill_attention_backend="fa3", + prefill_attention_backend="fa4", decode_attention_backend=None, disable_radix_cache=False, cuda_graph_config=CudaGraphConfig( @@ -216,12 +217,12 @@ class TestGemma4Renoise(unittest.TestCase): Gemma4Renoise.configure_server_args(server_args) self.assertEqual(server_args.attention_backend, "flashinfer") - self.assertEqual(server_args.prefill_attention_backend, "fa3") + self.assertEqual(server_args.prefill_attention_backend, "fa4") self.assertIsNone(server_args.decode_attention_backend) resolved = resolving_view(server_args) self.assertTrue(resolved.disable_radix_cache) self.assertEqual(resolved.cuda_graph_config.decode.backend, Backend.FULL) - self.assertEqual(resolved.cuda_graph_config.prefill.backend, Backend.DISABLED) + self.assertEqual(resolved.cuda_graph_config.prefill.backend, Backend.FULL) self.assertEqual(resolved.chunked_prefill_size, -1) self.assertFalse(server_args.disable_radix_cache) self.assertEqual(server_args.cuda_graph_config.decode.backend, Backend.FULL) @@ -299,7 +300,9 @@ class TestGemma4Renoise(unittest.TestCase): repeat = algorithm.init_step_state(_batch(["other-a", "other-b"], 4)) for first, second in zip(states, repeat): torch.testing.assert_close(first["current"], second["current"]) - torch.testing.assert_close(first["rng_state"], second["rng_state"]) + torch.testing.assert_close( + first["generator"].get_state(), second["generator"].get_state() + ) def test_entropy_bound_and_state_progression(self): algorithm = Gemma4Renoise( @@ -323,7 +326,7 @@ class TestGemma4Renoise(unittest.TestCase): log_probabilities = torch.log_softmax(processed, dim=-1) probabilities = log_probabilities.exp() generator = torch.Generator(device="cpu") - generator.set_state(state["rng_state"]) + generator.set_state(state["generator"].get_state()) sampled = torch.multinomial( probabilities, num_samples=1, generator=generator ).squeeze(-1) @@ -386,7 +389,10 @@ class TestGemma4Renoise(unittest.TestCase): for rid in first_by_rid: left, right = first_by_rid[rid], second_by_rid[rid] self.assertEqual(left["step"], right["step"]) - for key in ("current", "argmax", "self_conditioning", "rng_state"): + torch.testing.assert_close( + left["generator"].get_state(), right["generator"].get_state() + ) + for key in ("current", "argmax", "self_conditioning"): torch.testing.assert_close(left[key], right[key]) def test_max_step_completion_is_idempotent(self): diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index d001be9a81..bd3915465b 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -172,7 +172,13 @@ def test_import_stays_metadata_only(): "import sys, sglang.kernels.ops; " "print('DIRTY' if any(m in sys.modules for m in " "('sgl_kernel', 'cutlass', 'flydsl', 'aiter', " - "'sglang.kernels.ops.gemm.kimi_k3')) or any(" + "'sglang.kernels.ops.gemm.kimi_k3', " + "'sglang.kernels.ops.activation.softcap', " + "'sglang.kernels.ops.attention.dllm_kv_pack', " + "'sglang.kernels.ops.attention.flash_attention_v4', " + "'sglang.kernels.ops.attention.gemma_qkv_norm_rope', " + "'sglang.kernels.ops.layernorm.rmsnorm_fanout', " + "'sglang.kernels.ops.speculative.row_argmax')) or any(" "m.startswith('sglang.kernels.jit') for m in sys.modules) else 'CLEAN')" ) r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) @@ -239,6 +245,8 @@ def test_reclassified_public_entry_points_are_inventoried(): root = Path(K.__file__).resolve().parent / "ops" targets = {spec.target for spec in K.registry.all_specs()} modules = ( + "attention.dllm_kv_pack", + "attention.gemma_qkv_norm_rope", "attention.minicpm_sala.get_block_table", "attention.fast_topk", "attention.dsa.kpool_topk_transform", @@ -256,12 +264,14 @@ def test_reclassified_public_entry_points_are_inventoried(): "gemm.fp8_blockwise_gemm", "gemm.gptq_marlin", "layernorm.rmsnorm_hf", + "layernorm.rmsnorm_fanout", "layernorm.grouped_gemma_rmsnorm", "moe.dsv4", "moe.gemma4_routing", "memory.adler32", "memory.row_compact", "mm.process.image", + "speculative.row_argmax", ) for module in modules: tree = ast.parse( diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 0524cd87de..6396ab2d1e 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -3243,7 +3243,9 @@ class TestGoldenModelOverrides(_IsolatedPublish): page_size=1, # `use_mla_backend` reads the model configuration; a non-MLA # one keeps these assertions about the page constraints. - _model_config=SimpleNamespace(attention_arch=None), + _model_config=SimpleNamespace( + attention_arch=None, hf_config=SimpleNamespace(architectures=[]) + ), ) defaults.update(kw) return ResolvedView(SimpleNamespace(**defaults))