[Perf] Accelerate DiffusionGemma with FA4 and CUDA graphs on Blackwell (#42239)

Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
leonardHONG
2026-10-07 10:31:00 +08:00
committed by GitHub
co-authored by lianghougong Xiaoyu Zhang
parent 2f6e0ec45d
commit d7190b0122
43 changed files with 2192 additions and 181 deletions
@@ -344,6 +344,20 @@ for _fn in ("softcap_out", "softcap_inplace_logits"):
del _fn
register_kernel(
KernelSpec(
op="activation.softcap_to_float32_logits",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits",
capabilities=_CUDA,
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16", "float32"),
description="Softcap logits into a caller-provided FP32 output.",
),
)
)
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
register_kernel(
KernelSpec(
@@ -75,6 +75,9 @@ def softcap_inplace_logits_kernel(
ncols,
row_stride,
BLOCK_SIZE: tl.constexpr,
output_ptr=None,
output_row_stride=0,
CAST_TO_FP32: tl.constexpr = False,
):
row = tl.program_id(1).to(tl.int64)
pid = tl.program_id(0).to(tl.int64)
@@ -85,18 +88,24 @@ def softcap_inplace_logits_kernel(
# Load values
row_ptr = full_logits_ptr + row * row_stride
x = tl.load(row_ptr + offsets, mask=mask)
if CAST_TO_FP32:
x = x.to(tl.float32)
# Perform operations in-place
x = x / softcapping_value
x = libdevice.tanh(x)
x = x * softcapping_value
# Store result
tl.store(row_ptr + offsets, x, mask=mask)
if CAST_TO_FP32:
tl.store(output_ptr + row * output_row_stride + offsets, x, mask=mask)
else:
tl.store(row_ptr + offsets, x, mask=mask)
def softcap_inplace_logits(full_logits, final_logit_softcapping):
if full_logits.is_contiguous():
def _softcap_logits(full_logits, final_logit_softcapping, output=None):
if output is not None:
assert output.shape == full_logits.shape and output.dtype == torch.float32
if full_logits.is_contiguous() and (output is None or output.is_contiguous()):
nrows, ncols = 1, full_logits.numel()
row_stride = ncols
else:
@@ -106,6 +115,10 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping):
)
nrows, ncols = full_logits.shape
row_stride = full_logits.stride(0)
if output is not None:
assert output.stride(1) == 1 and (
nrows <= 1 or output.stride(0) >= ncols
), "softcap output requires contiguous, non-overlapping rows"
BLOCK_SIZE = 1024
grid = ((ncols + BLOCK_SIZE - 1) // BLOCK_SIZE, nrows)
@@ -116,5 +129,19 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping):
ncols=ncols,
row_stride=row_stride,
BLOCK_SIZE=BLOCK_SIZE,
output_ptr=output,
output_row_stride=(ncols if nrows == 1 else output.stride(0))
if output is not None
else 0,
CAST_TO_FP32=output is not None,
)
return full_logits
return full_logits if output is None else output
def softcap_inplace_logits(full_logits, final_logit_softcapping):
return _softcap_logits(full_logits, final_logit_softcapping)
def softcap_to_float32_logits(logits, final_logit_softcapping, output):
"""Write softcapped FP32 logits to an equally shaped, row-contiguous output."""
return _softcap_logits(logits, final_logit_softcapping, output)
@@ -360,6 +360,43 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="attention.pack_prefix_current",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current",
capabilities=frozenset({CapabilityRequirement.CUDA}),
description="Pack CSR prefix and current K/V into caller-provided dense buffers.",
)
)
register_kernel(
KernelSpec(
op="attention.flash_attn_gqa_512",
backend=KernelBackend.CUTE_DSL,
target="sglang.kernels.ops.attention.flash_attention_v4:flash_attn_gqa_512",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))}
),
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16"),
description="SM100/SM110 D512 GQA with separate K/V and optional page tables.",
),
)
)
register_kernel(
KernelSpec(
op="attention.gemma_qkv_norm_rope",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope",
capabilities=frozenset({CapabilityRequirement.CUDA}),
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16"),
in_place=True,
description="Strided QKV normalization and partial NeoX RoPE.",
),
)
)
for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
register_kernel(
KernelSpec(
@@ -0,0 +1,93 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0.
"""Pack a CSR prefix and current K/V into capture-stable dense storage."""
import triton
import triton.language as tl
@triton.jit
def _pack_kv(
K,
V,
KB,
VB,
QO,
KI,
IDS,
KD,
VD,
CU,
H: tl.constexpr,
D: tl.constexpr,
KS: tl.constexpr,
KH: tl.constexpr,
VS: tl.constexpr,
VH: tl.constexpr,
KBS: tl.constexpr,
KBH: tl.constexpr,
VBS: tl.constexpr,
VBH: tl.constexpr,
TILE: tl.constexpr,
):
row = tl.program_id(0)
qs = tl.load(QO + row).to(tl.int64)
qe = tl.load(QO + row + 1).to(tl.int64)
ps = tl.load(KI + row).to(tl.int64)
pe = tl.load(KI + row + 1).to(tl.int64)
prefix = pe - ps
length = prefix + qe - qs
start = ps + qs
offsets = tl.program_id(1).to(tl.int64) * TILE + tl.arange(0, TILE)
token, head, channel = offsets // (H * D), offsets // D % H, offsets % D
slot = tl.load(IDS + ps + token, token < prefix, other=0).to(tl.int64)
kp = tl.load(KB + slot * KBS + head * KBH + channel, token < prefix, other=0)
vp = tl.load(VB + slot * VBS + head * VBH + channel, token < prefix, other=0)
current = qs + token - prefix
kc = tl.load(
K + current * KS + head * KH + channel,
(token >= prefix) & (token < length),
other=0,
)
vc = tl.load(
V + current * VS + head * VH + channel,
(token >= prefix) & (token < length),
other=0,
)
tl.store(
KD + start * H * D + offsets, tl.where(token < prefix, kp, kc), token < length
)
tl.store(
VD + start * H * D + offsets, tl.where(token < prefix, vp, vc), token < length
)
if tl.program_id(1) == 0:
tl.store(CU + row, start)
if row == tl.num_programs(0) - 1:
tl.store(CU + row + 1, pe + qe)
def pack_prefix_current(k, v, kb, vb, qo, ki, ids, kd, vd, cu, max_length):
h, d = k.shape[1:]
_pack_kv[(qo.numel() - 1, triton.cdiv(max_length * h * d, 1024))](
k,
v,
kb,
vb,
qo,
ki,
ids,
kd,
vd,
cu,
h,
d,
k.stride(0),
k.stride(1),
v.stride(0),
v.stride(1),
kb.stride(0),
kb.stride(1),
vb.stride(0),
vb.stride(1),
TILE=1024,
)
@@ -27,12 +27,7 @@ from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides
from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd
from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors
from sglang.srt.environ import envs
from sglang.srt.utils import (
is_cuda,
is_gfx95_supported,
is_gfx1250_supported,
is_hip,
)
from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip
_is_cuda = is_cuda()
if _is_cuda:
@@ -1257,8 +1252,14 @@ def _fwd_kernel_unified(
deno = tl.zeros([BLOCK_M], dtype=tl.float32)
e_max = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
# Unified loop: process all KV tokens (prefix + extend)
for start_n in range(0, cur_seq_kv_len, BLOCK_N):
start_k, end_k = 0, cur_seq_kv_len
if IS_CAUSAL and SLIDING_WINDOW_SIZE > 0 and not USE_CUSTOM_MASK:
first_query = cur_block_m * BLOCK_M
last_query = tl.minimum(first_query + BLOCK_M, cur_seq_q_len)
start_k = tl.maximum(0, cur_seq_prefix_len + first_query - SLIDING_WINDOW_SIZE)
start_k = start_k // BLOCK_N * BLOCK_N
end_k = tl.minimum(cur_seq_kv_len, cur_seq_prefix_len + last_query)
for start_n in range(start_k, end_k, BLOCK_N):
start_n = tl.multiple_of(start_n, BLOCK_N)
mask_n = (start_n + offs_n) < cur_seq_kv_len
@@ -1,6 +1,7 @@
from __future__ import annotations
import os
from functools import lru_cache
from typing import Callable, Optional, Tuple, Union
import torch
@@ -28,6 +29,102 @@ def is_flash_attention_v4_available() -> bool:
return _flash_attn_varlen_func is not None
@lru_cache(maxsize=1)
def _get_gqa_512_jit_cache():
from sglang.kernels.ops.attention.flash_attn.cute.interface import _get_jit_cache
return _get_jit_cache("fwd_gqa_512")
def flash_attn_gqa_512(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
out: torch.Tensor,
*,
cu_seqlens_q: Optional[torch.Tensor] = None,
cu_seqlens_k: Optional[torch.Tensor] = None,
seqused_k: Optional[torch.Tensor] = None,
page_table: Optional[torch.Tensor] = None,
softmax_scale: float = 1.0,
lse: Optional[torch.Tensor] = None,
pack_gqa: bool = True,
causal: bool = False,
) -> torch.Tensor:
"""GQA with 512-dimensional keys and separate values."""
import cutlass.cute as cute
from sglang.kernels.ops.attention.flash_attn.cute.cute_dsl_utils import (
to_cute_tensor,
)
from sglang.kernels.ops.attention.flash_attn.cute.flash_fwd_mla_sm100 import (
FlashAttentionMLAForwardSm100,
)
if cu_seqlens_q is not None:
cu_seqlens_q = cu_seqlens_q.to(dtype=torch.int32)
args = (
q,
k,
v,
out,
lse,
softmax_scale,
cu_seqlens_q,
cu_seqlens_k,
seqused_k,
page_table,
)
use_cpasync = page_table is not None and k.shape[1] != 128
key = (
pack_gqa,
causal,
q.device,
q.shape[-2] // k.shape[-2],
k.shape[-2],
q.shape[-1],
v.shape[-1],
use_cpasync,
tuple(
(t.ndim, t.dtype, tuple(s if s in (0, 1) else 2 for s in t.stride()))
if isinstance(t, torch.Tensor)
else t
for t in args
),
)
cache = _get_gqa_512_jit_cache()
if key not in cache:
compile_args = [
to_cute_tensor(
t,
assumed_align=4
if t.dtype in (torch.int32, torch.int64, torch.float32)
else 16,
)
if isinstance(t, torch.Tensor)
else t
for t in args
]
kernel = FlashAttentionMLAForwardSm100(
is_causal=causal,
use_cpasync_load_KV=use_cpasync,
is_topk_gather=False,
pack_gqa=pack_gqa,
qhead_per_kvhead=q.shape[-2] // k.shape[-2],
nheads_kv=k.shape[-2],
is_varlen_q=cu_seqlens_q is not None,
has_qk=False,
)
cache[key] = cute.compile(
kernel.forward_gqa,
*compile_args,
stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
options="--enable-tvm-ffi",
)
cache[key](*args)
return out
def _maybe_contiguous(x: Optional[torch.Tensor]) -> Optional[torch.Tensor]:
return x.contiguous() if x is not None and x.stride(-1) != 1 else x
@@ -353,6 +353,28 @@ class FlashAttentionMLAForwardSm100:
return SharedStorage
@cute.jit
def forward_gqa(
self,
mQ: cute.Tensor,
mK: cute.Tensor,
mValue: cute.Tensor,
mO: cute.Tensor,
mLSE: Optional[cute.Tensor],
softmax_scale: Float32,
mCuSeqlensQ: Optional[cute.Tensor] = None,
mCuSeqlensK: Optional[cute.Tensor] = None,
mSeqUsedK: Optional[cute.Tensor] = None,
mPageTable: Optional[cute.Tensor] = None,
stream: cuda.CUstream = None,
):
self(
None, mQ, None, mK, mO, mLSE, softmax_scale,
mCuSeqlensQ=mCuSeqlensQ, mCuSeqlensK=mCuSeqlensK,
mSeqUsedK=mSeqUsedK, mPageTable=mPageTable,
stream=stream, mValue=mValue,
)
# fmt: off
@cute.jit
def __call__(
@@ -374,12 +396,14 @@ class FlashAttentionMLAForwardSm100:
mPageTable: Optional[cute.Tensor] = None,
window_size_left: Int32 | int | None = None,
window_size_right: Int32 | int | None = None,
# Always keep stream as the last parameter (EnvStream: obtained implicitly via TVM FFI).
stream: cuda.CUstream = None,
*,
mValue: Optional[cute.Tensor] = None,
):
# fmt: on
self.store_P = mP is not None
self.store_row_max = mRowMax is not None
self.separate_value = mValue is not None
if const_expr(self.has_qk):
assert mQ is not None and mK is not None, "has_qk requires mQ and mK"
@@ -402,11 +426,11 @@ class FlashAttentionMLAForwardSm100:
*(cute.assume(s, divby=128 // mX.element_type.width) for s in mX.stride[:-1]),
mX.stride[-1],
)
mQ, mQv, mK, mV, mO, mP = [
mQ, mQv, mK, mV, mO, mP, mValue = [
cute.make_tensor(mX.iterator, cute.make_layout(mX.shape, stride=new_stride(mX)))
if mX is not None
else None
for mX in (mQ, mQv, mK, mV, mO, mP)
for mX in (mQ, mQv, mK, mV, mO, mP, mValue)
]
# (b, s, h, d) -> (s, d, h, b) or
@@ -420,16 +444,18 @@ class FlashAttentionMLAForwardSm100:
else None
for mX in (mQ, mQv, mO, mP)
]
mK, mV = [
mK, mV, mValue = [
cute.make_tensor(mX.iterator, cute.select(mX.layout, mode=KV_layout_transpose))
if mX is not None
else None
for mX in (mK, mV)
for mX in (mK, mV, mValue)
]
# (s_k, dv, h_k, b) -> (dv, s_k, h_k, b) or
# (total_k, dv, h_k) -> (dv, total_k, h_k)
V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2]
mVt = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose))
# MLA uses the same latent for scores and values; GQA supplies separate values.
value = mV if const_expr(mValue is None) else mValue
mVt = cute.make_tensor(value.iterator, cute.select(value.layout, mode=V_layout_transpose))
# (b, s_q, topk) -> (topk, s_q, b) or (total_q, topk) -> (topk, total_q)
topk_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0]
mIndexTopk = (
@@ -2605,7 +2631,9 @@ class FlashAttentionMLAForwardSm100:
softmax = SoftmaxSm100.create(
softmax_scale_log2,
rescale_threshold=8.0 if const_expr(self.dtype_Q.width == 16) else 0.0,
rescale_threshold=8.0
if const_expr(self.dtype_Q.width == 16 and not self.separate_value)
else 0.0,
softmax_scale=softmax_scale,
)
softmax.reset()
@@ -341,9 +341,17 @@ class BlackwellFusedMultiHeadAttentionForward:
stride_b_lse = h_r64 * h_k64 * s_lse64 if cum_seqlen_q is None else 0
# (s, d, ((h_r, h_k), b))
if cutlass.const_expr(q_rank == 5):
q_row_stride = Int64(q_tensor.stride[1])
q_head_stride = (Int64(q_tensor.stride[3]), Int64(q_tensor.stride[2]))
else:
q_row_stride = Int64(q_tensor.stride[q_rank - 3])
head_stride = Int64(q_tensor.stride[q_rank - 2])
q_head_stride = (head_stride, head_stride * h_r64)
q_batch_stride = Int64(q_tensor.stride[0]) if cum_seqlen_q is None else 0
q_layout = cute.make_layout(
(s_q_total, d, ((h_r, h_k), b)),
stride=(d64 * h_r64 * h_k64, 1, ((d64, d64 * h_r64), stride_b_qo)),
stride=(q_row_stride, 1, (q_head_stride, q_batch_stride)),
)
q = cute.make_tensor(q_tensor.iterator, q_layout)
if cutlass.const_expr(mPageTable is not None):
@@ -12,6 +12,7 @@ from typing import Optional, Tuple
import torch
import triton
import triton.language as tl
from triton.language.extra.cuda import libdevice
def _pdl_supported() -> bool:
@@ -60,6 +61,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
HAS_GATE: tl.constexpr,
MROPE: tl.constexpr,
ENABLE_PDL: tl.constexpr,
WEIGHT_SHIFT: tl.constexpr = 1.0,
EXPLICIT_ROPE_FMA: tl.constexpr = False,
):
token = tl.program_id(0)
head = tl.program_id(1)
@@ -86,7 +89,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
var = tl.sum(x * x, axis=0) / HEAD_DIM
inv_rms = tl.rsqrt(var + EPS)
x_norm = (x * inv_rms * (w + 1.0)).to(out_dtype).to(tl.float32)
x_norm = (x * inv_rms * (w + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
# Pass-through tail [rotary_dim, head_dim)
if HAS_PASS:
@@ -104,8 +107,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
wr2 = tl.load(w_ptr + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to(
tl.float32
)
xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(out_dtype).to(tl.float32)
xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(out_dtype).to(tl.float32)
xr1 = (xr1 * inv_rms * (wr1 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
xr2 = (xr2 * inv_rms * (wr2 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
if MROPE:
axis = tl.load(mrope_axis_map_ptr + rot_offs, mask=rot_mask, other=0)
@@ -121,8 +124,14 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
sin = tl.load(
cos_sin_cache_ptr + cache_off + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0
).to(tl.float32)
tl.store(out_base + rot_offs, (xr1 * cos - xr2 * sin), mask=rot_mask)
tl.store(out_base + HALF_ROTARY + rot_offs, (xr2 * cos + xr1 * sin), mask=rot_mask)
if EXPLICIT_ROPE_FMA:
first = tl.fma(xr1, cos, -libdevice.mul_rn(xr2, sin))
second = tl.fma(xr2, cos, libdevice.mul_rn(xr1, sin))
else:
first = xr1 * cos - xr2 * sin
second = xr2 * cos + xr1 * sin
tl.store(out_base + rot_offs, first, mask=rot_mask)
tl.store(out_base + HALF_ROTARY + rot_offs, second, mask=rot_mask)
# Gate copy (Q heads only)
if HAS_GATE and not is_k:
@@ -0,0 +1,124 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0.
"""In-place Gemma QKV normalization and partial NeoX RoPE."""
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import (
_fused_qk_rmsnorm_rope_gate_kernel,
)
from sglang.kernels.ops.layernorm.gemma4_fused_ops import _gemma_qkv_rmsnorm_store
@triton.jit
def _gemma_qkv_norm_rope(
Q,
K,
V,
QW,
KW,
CACHE,
POS,
SQ: tl.constexpr,
SK: tl.constexpr,
SV: tl.constexpr,
SC: tl.constexpr,
HQ: tl.constexpr,
HK: tl.constexpr,
D: tl.constexpr,
R: tl.constexpr,
EPS: tl.constexpr,
FP16: tl.constexpr,
BLOCK: tl.constexpr,
ROT_BLOCK: tl.constexpr,
):
head = tl.program_id(1)
if head < HQ + HK:
_fused_qk_rmsnorm_rope_gate_kernel(
Q,
K,
Q,
K,
Q,
QW,
KW,
CACHE,
POS,
POS,
SQ,
SK,
SQ,
SK,
SQ,
SC,
1,
HQ,
HK,
D,
R,
R // 2,
BLOCK,
ROT_BLOCK,
EPS,
FP16,
R < D,
False,
False,
False,
WEIGHT_SHIFT=0.0,
EXPLICIT_ROPE_FMA=True,
)
else:
cols = tl.arange(0, BLOCK)
_gemma_qkv_rmsnorm_store(
V,
QW,
SV,
tl.program_id(0),
head - HQ - HK,
cols,
cols < D,
D,
EPS,
False,
)
def gemma_qkv_norm_rope(
q,
k,
v,
q_weight,
k_weight,
cache,
positions,
num_q_heads,
num_kv_heads,
head_dim,
rotary_dim,
eps,
):
"""Normalize Q/K with weights and V without weights, then rotate Q/K."""
_gemma_qkv_norm_rope[(q.shape[0], num_q_heads + 2 * num_kv_heads)](
q,
k,
v,
q_weight,
k_weight,
cache,
positions,
q.stride(0),
k.stride(0),
v.stride(0),
cache.stride(0),
num_q_heads,
num_kv_heads,
head_dim,
rotary_dim,
eps,
q.dtype == torch.float16,
triton.next_power_of_2(head_dim),
triton.next_power_of_2(rotary_dim // 2),
)
@@ -33,6 +33,7 @@ def create_flashinfer_kv_indices_triton(
req_to_token_ptr_stride,
ENTRY_PAGE_SIZE: tl.constexpr = 1,
TOKEN_BLOCK_PARALLEL: tl.constexpr = False,
token_mapping=None,
):
"""Gather per-request token ids into a flat CSR kv_indices stream.
@@ -48,6 +49,7 @@ def create_flashinfer_kv_indices_triton(
(which bottlenecks long-context spec decode, where this kernel runs every
iteration). With the default, the kernel is the historical
one-program-per-request loop and 1D launch sites are unaffected.
``token_mapping`` remaps gathered token ids in the same pass, preserving -1.
"""
BLOCK_SIZE: tl.constexpr = 512
pid = tl.program_id(axis=0)
@@ -95,6 +97,8 @@ def create_flashinfer_kv_indices_triton(
mask=mask,
)
data = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
if token_mapping is not None:
data = tl.load(token_mapping + data, mask=mask & (data >= 0), other=-1)
tl.store(kv_indices_ptr + kv_indices_offset + offset, data, mask=mask)
@@ -580,3 +580,18 @@ register_kernel(
target="sglang.kernels.ops.layernorm.rms_normalize_hip:rms_normalize_triton",
)
)
register_kernel(
KernelSpec(
op="layernorm.rmsnorm_fanout",
backend=KernelBackend.CUTE_DSL,
target="sglang.kernels.ops.layernorm.rmsnorm_fanout:rmsnorm_fanout",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
),
format_signature=FormatSignature(
supported_dtypes=_NORM_DTYPES,
description="Three weighted RMSNorm outputs sharing a single-CTA row reduction.",
),
)
)
@@ -10,6 +10,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_sm100_supported
@triton.jit
def _gemma_rmsnorm_residual_kernel(
@@ -277,7 +279,7 @@ def gemma_qkv_rmsnorm(
assert k.stride(-1) == 1 and v.stride(-1) == 1
assert k_weight is not None and k_weight.shape[-1] == head_dim
if M <= 256:
if M <= 256 or (M == 512 and is_sm100_supported()):
total_heads = num_q_heads + (2 * num_kv_heads if has_kv else 0)
_gemma_qkv_rmsnorm_kernel[(M, total_heads)](
q,
@@ -0,0 +1,211 @@
# Copyright 2026 SGLang Team
# Copyright 2025 FlashInfer team
# Licensed under the Apache License, Version 2.0.
"""Three RMSNorm outputs sharing FlashInfer's input load and reduction.
Uses the CuTe RMSNormKernel layout/reduction interface from FlashInfer 0.6.18.
The row-count dispatch covers the measured decode buckets on Blackwell.
"""
from functools import cache
import torch
_FANOUT_MAX_ROWS = 2048
def can_use_rmsnorm_fanout(x):
return (
0 < x.shape[0] <= _FANOUT_MAX_ROWS
and _fanout_kernel(x.dtype, x.shape[1]) is not None
)
@cache
def _fanout_kernel(dtype, width):
if dtype not in (torch.bfloat16, torch.float16):
return None
kernel = _fanout_kernel_type()(_cutlass_dtype(dtype), width)
return kernel if kernel.cluster_n == 1 else None
def _cutlass_dtype(dtype):
import cutlass
return cutlass.BFloat16 if dtype == torch.bfloat16 else cutlass.Float16
def rmsnorm_fanout(x, weight0, weight1, weight2, eps):
"""Normalize contiguous FP16/BF16 rows with three matching [H] weights."""
outputs = tuple(torch.empty_like(x) for _ in range(3))
compiled = _compile_fanout(x.device, x.dtype, x.shape[1])
compiled(x, weight0, weight1, weight2, *outputs, x.shape[0], eps)
return outputs
@cache
def _compile_fanout(device, torch_dtype, width):
import cutlass
import cutlass.cute as cute
dtype = _cutlass_dtype(torch_dtype)
kernel = _fanout_kernel(torch_dtype, width)
rows = cute.sym_int(64)
matrix = cute.runtime.make_fake_compact_tensor(
dtype, (rows, width), stride_order=(1, 0), assumed_align=16
)
weight = cute.runtime.make_fake_compact_tensor(dtype, (width,), assumed_align=16)
return cute.compile(
kernel,
matrix,
weight,
weight,
weight,
matrix,
matrix,
matrix,
cutlass.Int64(1),
cutlass.Float32(1e-6),
stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
options="--enable-tvm-ffi",
)
@cache
def _fanout_kernel_type():
import cutlass
import cutlass.cute as cute
from cutlass import Float32, Int64
from flashinfer.norm.kernels.rmsnorm import RMSNormKernel
from flashinfer.norm.utils import predicate_k, row_reduce_sum_multirow
class RMSNormFanout(RMSNormKernel):
@cute.jit
def __call__(
self,
x: cute.Tensor,
w0: cute.Tensor,
w1: cute.Tensor,
w2: cute.Tensor,
y0: cute.Tensor,
y1: cute.Tensor,
y2: cute.Tensor,
m: Int64,
eps: Float32,
stream,
):
shape, stride = self._make_tv_layout(
self.threads_per_row,
self.rows_per_block,
self.vec_size,
self.num_vec_blocks,
)
layout = cute.make_layout(shape, stride=stride)
tile = (self.rows_per_block, self.cols_per_tile)
self.kernel(x, w0, w1, w2, y0, y1, y2, m, eps, layout, tile).launch(
grid=[cute.ceil_div(m, self.rows_per_block), 1, 1],
block=[self.num_threads, 1, 1],
smem=self._smem_size_in_bytes(),
stream=stream,
)
@cute.kernel
def kernel(
self,
x: cute.Tensor,
w0: cute.Tensor,
w1: cute.Tensor,
w2: cute.Tensor,
y0: cute.Tensor,
y1: cute.Tensor,
y2: cute.Tensor,
m: Int64,
eps: Float32,
layout: cute.Layout,
tile: cute.Shape,
):
tid, _, _ = cute.arch.thread_idx()
block, _, _ = cute.arch.block_idx()
smem = cutlass.utils.SmemAllocator()
if cutlass.const_expr(self.use_async_copy):
sx = smem.allocate_tensor(
x.element_type,
cute.make_ordered_layout(tile, order=(1, 0)),
byte_alignment=16,
)
reduction = smem.allocate_tensor(
Float32,
cute.make_layout((self.rows_per_block, self.warps_per_row)),
byte_alignment=4,
)
gx = cute.local_tile(x, tile, (block, 0))
coords = cute.local_tile(
cute.make_identity_tensor(x.shape), tile, (block, 0)
)
sync = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(),
x.element_type,
num_bits_per_copy=self.copy_bits,
)
if cutlass.const_expr(self.use_async_copy):
async_copy = cute.make_copy_atom(
cute.nvgpu.cpasync.CopyG2SOp(),
x.element_type,
num_bits_per_copy=self.copy_bits,
)
loader = cute.make_tiled_copy(async_copy, layout, tile)
else:
loader = cute.make_tiled_copy(sync, layout, tile)
copier = cute.make_tiled_copy(sync, layout, tile)
thread_x = loader.get_slice(tid)
thread_o = copier.get_slice(tid)
txg = thread_x.partition_S(gx)
txc = thread_x.partition_S(coords)
txr = cute.make_fragment_like(txg)
x_pred = predicate_k(txc, limit=self.H)
row_valid = txc[(0, 0), 0, 0][0] < m
if cutlass.const_expr(self.use_async_copy):
txs = thread_x.partition_D(sx)
if row_valid:
cute.copy(async_copy, txg, txs, pred=x_pred)
cute.arch.cp_async_commit_group()
cute.arch.cp_async_wait_group(0)
cute.autovec_copy(txs, txr)
else:
txr.store(cute.zeros_like(txr, dtype=x.element_type))
if row_valid:
cute.copy(sync, txg, txr, pred=x_pred)
value = txr.load().to(Float32)
summed = row_reduce_sum_multirow(
value * value, self.threads_per_row, reduction, None, 1
)
rstd = cute.math.rsqrt(summed / Float32(self.H) + eps, fastmath=True)
cute.arch.barrier()
if cutlass.const_expr(self.use_async_copy):
cute.autovec_copy(txs, txr)
value = txr.load().to(Float32)
weights = (w0, w1, w2)
outputs = (y0, y1, y2)
wpred = predicate_k(thread_o.partition_S(coords), limit=self.H)
for index in cutlass.range_constexpr(3):
weight = weights[index]
expanded = cute.make_tensor(
weight.iterator,
cute.prepend(
weight.layout, cute.make_layout((tile[0],), stride=(0,))
),
)
gw = cute.local_tile(expanded, tile, (0, 0))
twg = thread_o.partition_S(gw)
twr = cute.make_fragment_like(twg)
cute.copy(sync, twg, twr, pred=wpred)
weight_value = thread_x.retile(twr).load().to(Float32)
gy = cute.local_tile(outputs[index], tile, (block, 0))
tog = thread_o.partition_D(gy)
tor = cute.make_fragment_like(tog)
result = value * rstd * (weight_value + Float32(0.0))
tor.store(result.to(outputs[index].element_type))
if row_valid:
cute.copy(sync, tor, tog, pred=x_pred)
return RMSNormFanout
@@ -111,3 +111,17 @@ register_kernel(
target="sglang.kernels.ops.sampling.murmur_hash:murmur_hash32",
)
)
register_kernel(
KernelSpec(
op="sampling.denoiser_statistics",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.sampling.denoiser_statistics:denoiser_statistics",
capabilities=frozenset({CapabilityRequirement.CUDA}),
format_signature=FormatSignature(
supported_dtypes=("float32",),
description="FP32 probabilities, entropy and raw-logit argmax for contiguous [B,M,V] logits; optional BF16 probabilities.",
),
)
)
@@ -0,0 +1,197 @@
"""Chunked FP32 statistics for large-vocabulary block denoising.
The partials carry a maximum, exponential sum and centered first moment.
Merging these computes entropy without materializing log-probabilities. A
separate pass writes FP32 probabilities for the existing categorical sampler
and self-conditioning GEMM. Sampling and its generator state stay unchanged.
"""
import torch
import triton
import triton.language as tl
@triton.jit
def _denoiser_partial_statistics(
LOGITS,
TEMPERATURES,
PARTIALS,
PARTIAL_ARGMAX,
VOCAB: tl.constexpr,
CANVAS: tl.constexpr,
ROWS: tl.constexpr,
CHUNKS: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
part = tl.program_id(1)
j = part * BLOCK + tl.arange(0, BLOCK)
temp = tl.load(TEMPERATURES + row // CANVAS)
raw = tl.load(
LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf")
).to(tl.float32)
# Approximate reciprocal scaling changes sharp distributions enough to
# exceed the statistics tolerance. Match FP32 division rounding.
scaled = tl.div_rn(raw, temp)
mx = tl.max(scaled, 0)
d = scaled - tl.where(mx == -float("inf"), 0.0, mx)
e = tl.exp(d)
z = tl.sum(e, 0)
s = tl.sum(tl.where(j < VOCAB, d * e, 0.0), 0)
# Preserve PyTorch argmax's first-NaN and first-maximum behavior.
# An explicit NaN reduction is required; testing the reduced maximum
# itself is not equivalent after Triton lowering.
rawmax = tl.max(raw, 0)
has_nan = tl.sum(((j < VOCAB) & (raw != raw)).to(tl.int32), 0) > 0
idx = tl.min(
tl.where(
(j < VOCAB) & tl.where(has_nan, raw != raw, raw == rawmax), j, 2147483647
),
0,
)
off = row * CHUNKS + part
tl.store(PARTIALS + off, mx)
tl.store(PARTIALS + ROWS * CHUNKS + off, z)
tl.store(PARTIALS + 2 * ROWS * CHUNKS + off, s)
tl.store(PARTIALS + 3 * ROWS * CHUNKS + off, rawmax)
tl.store(PARTIAL_ARGMAX + off, idx)
@triton.jit
def _denoiser_merge_statistics(
PARTIALS,
PARTIAL_ARGMAX,
NORMALIZER,
ENTROPY,
ARGMAX,
ROWS: tl.constexpr,
CHUNKS: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
p = tl.arange(0, BLOCK)
off = row * CHUNKS + p
mx = tl.load(PARTIALS + off, mask=p < CHUNKS, other=-float("inf"))
z = tl.load(PARTIALS + ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0)
moment = tl.load(PARTIALS + 2 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0)
maximum = tl.max(mx, 0)
shift = mx - maximum
alpha = tl.exp(shift)
denom = tl.sum(alpha * z, 0)
total = tl.sum(
tl.where(p < CHUNKS, alpha * (moment + tl.where(z > 0, z * shift, 0.0)), 0.0), 0
)
logz = tl.log(denom)
tl.store(NORMALIZER + row, maximum)
tl.store(NORMALIZER + ROWS + row, logz)
tl.store(ENTROPY + row, logz - total / denom)
rawmax = tl.load(
PARTIALS + 3 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=-float("inf")
)
best = tl.max(rawmax, 0)
idx = tl.load(PARTIAL_ARGMAX + off, mask=p < CHUNKS, other=2147483647)
has_nan = tl.sum(((p < CHUNKS) & (rawmax != rawmax)).to(tl.int32), 0) > 0
tl.store(
ARGMAX + row,
tl.min(
tl.where(
(p < CHUNKS) & tl.where(has_nan, rawmax != rawmax, rawmax == best),
idx,
2147483647,
),
0,
),
)
@triton.jit
def _denoiser_write_probabilities(
LOGITS,
TEMPERATURES,
NORMALIZER,
PROBABILITIES,
SOFT_PROBABILITIES,
VOCAB: tl.constexpr,
CANVAS: tl.constexpr,
ROWS: tl.constexpr,
BLOCK: tl.constexpr,
WRITE_SOFT: tl.constexpr,
):
row = tl.program_id(0)
j = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
temp = tl.load(TEMPERATURES + row // CANVAS)
mx = tl.load(NORMALIZER + row)
logz = tl.load(NORMALIZER + ROWS + row)
raw = tl.load(
LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf")
).to(tl.float32)
p = tl.exp(tl.div_rn(raw, temp) - mx - logz)
tl.store(PROBABILITIES + row.to(tl.int64) * VOCAB + j, p, mask=j < VOCAB)
if WRITE_SOFT:
tl.store(
SOFT_PROBABILITIES + row.to(tl.int64) * VOCAB + j,
p.to(tl.bfloat16),
mask=j < VOCAB,
)
def denoiser_statistics(
logits: torch.Tensor,
temperatures: torch.Tensor,
soft_probabilities: torch.Tensor | None = None,
):
"""Statistics for contiguous FP32 [B, M, V] logits and [B] temperatures.
soft_probabilities, when supplied, is a contiguous BF16 buffer shaped like logits.
"""
assert logits.ndim == 3 and temperatures.shape == (logits.shape[0],)
block = 4096
batch, canvas, vocab = logits.shape
rows = batch * canvas
parts = triton.cdiv(vocab, block)
partials = torch.empty((4, rows, parts), device=logits.device, dtype=torch.float32)
partial_argmax = torch.empty((rows, parts), device=logits.device, dtype=torch.int32)
normalizer = torch.empty((2, rows), device=logits.device, dtype=torch.float32)
probabilities = torch.empty_like(logits, dtype=torch.float32)
entropies = torch.empty((batch, canvas), device=logits.device, dtype=torch.float32)
argmax = torch.empty((batch, canvas), device=logits.device, dtype=torch.int64)
_denoiser_partial_statistics[(rows, parts)](
logits,
temperatures,
partials,
partial_argmax,
vocab,
canvas,
rows,
parts,
block,
num_warps=4,
enable_fp_fusion=False,
)
_denoiser_merge_statistics[(rows,)](
partials,
partial_argmax,
normalizer,
entropies,
argmax,
rows,
parts,
triton.next_power_of_2(parts),
num_warps=4,
enable_fp_fusion=False,
)
_denoiser_write_probabilities[(rows, triton.cdiv(vocab, block))](
logits,
temperatures,
normalizer,
probabilities,
soft_probabilities if soft_probabilities is not None else probabilities,
vocab,
canvas,
rows,
block,
soft_probabilities is not None,
num_warps=4,
enable_fp_fusion=False,
)
return probabilities, entropies, argmax
@@ -6,7 +6,12 @@ The Triton kernels migrated here live in this package
"""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import KernelBackend, KernelSpec
from sglang.kernels.spec import (
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
)
# (module, public_fn) migrated from speculative/triton_ops.
_TRITON_KERNELS = [
@@ -37,4 +42,17 @@ for _mod, _fn in _TRITON_KERNELS:
)
del _mod, _fn
register_kernel(
KernelSpec(
op="speculative.div_argmax",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.speculative.row_argmax:div_argmax",
capabilities=frozenset({CapabilityRequirement.CUDA}),
format_signature=FormatSignature(
supported_dtypes=("float32",),
description="Row-wise division and argmax on matching 2D tensors with unit column strides.",
),
)
)
__all__ = []
@@ -68,11 +68,16 @@ def _medium_argmax_partial_kernel(
SX: tl.constexpr,
SPLITS: tl.constexpr,
BLOCK: tl.constexpr,
DIVISOR=None,
SD: tl.constexpr = 0,
):
row, part = tl.program_id(0), tl.program_id(1)
ix = part * BLOCK + tl.arange(0, BLOCK)
valid = ix < N
v = tl.load(X + row * SX + ix, valid, float("-inf"))
if DIVISOR is not None:
divisor = tl.load(DIVISOR + row * SD + ix, valid, 1.0)
v = tl.div_rn(v, divisor)
i = tl.where(valid, ix, N)
best_v, best_i = tl.reduce((v, i), 0, _argmax_pair)
tl.store(PV + row * SPLITS + part, best_v)
@@ -89,25 +94,50 @@ def _medium_argmax_final_kernel(PV, PI, OUT, SPLITS: tl.constexpr, BLOCK: tl.con
tl.store(OUT + row, index.to(tl.int64))
def div_argmax(x: torch.Tensor, divisor: torch.Tensor) -> torch.Tensor:
"""FP32 row-wise ``(x / divisor).argmax(-1)`` with unit column strides."""
assert x.ndim == 2 and x.dtype == torch.float32 and x.stride(1) == 1
assert (
divisor.shape == x.shape
and divisor.dtype == x.dtype
and divisor.device == x.device
and divisor.stride(1) == 1
)
return _medium_row_argmax(x, divisor)
def _medium_row_argmax(x, divisor=None):
rows, n = x.shape
block = 4096 if rows <= 256 else 8192
splits = triton.cdiv(n, block)
values = torch.empty((rows, splits), dtype=torch.float32, device=x.device)
indices = torch.empty((rows, splits), dtype=torch.int32, device=x.device)
out = torch.empty(rows, dtype=torch.int64, device=x.device)
_medium_argmax_partial_kernel[(rows, splits)](
x,
values,
indices,
n,
x.stride(0),
splits,
block,
DIVISOR=divisor,
SD=divisor.stride(0) if divisor is not None else 0,
num_warps=4,
)
_medium_argmax_final_kernel[(rows,)](
values, indices, out, splits, triton.next_power_of_2(splits), num_warps=1
)
return out
def row_argmax(x: torch.Tensor) -> torch.Tensor:
"""``x.argmax(dim=-1)`` for FP32 speculative logits with a wide vocab."""
assert x.dim() == 2 and x.dtype == torch.float32 and x.stride(1) == 1
rows, n = x.shape
out = torch.empty((rows,), dtype=torch.int64, device=x.device)
if rows > 64:
# Whole aligned tiles avoid the loop and unaligned partition starts.
# Larger batches use wider tiles to limit the total number of CTAs.
block = 4096 if rows <= 256 else 8192
splits = triton.cdiv(n, block)
pv = torch.empty((rows, splits), dtype=torch.float32, device=x.device)
pi = torch.empty((rows, splits), dtype=torch.int32, device=x.device)
_medium_argmax_partial_kernel[(rows, splits)](
x, pv, pi, n, x.stride(0), splits, block, num_warps=4
)
_medium_argmax_final_kernel[(rows,)](
pv, pi, out, splits, triton.next_power_of_2(splits), num_warps=1
)
return out
return _medium_row_argmax(x)
out = torch.empty((rows,), dtype=torch.int64, device=x.device)
pv = torch.empty((rows, _SPLITS), dtype=torch.float32, device=x.device)
pi = torch.empty((rows, _SPLITS), dtype=torch.int32, device=x.device)
_argmax_partial_kernel[(rows, _SPLITS)](
+2 -4
View File
@@ -26,11 +26,10 @@ def handle_dllm_inference(server_args: Any):
return
from sglang.srt.dllm.algorithm import get_algorithm_cls
run_post_process_pass(server_args, _dllm_attention_backend)
get_algorithm_cls(cfg.dllm_algorithm).configure_server_args(server_args)
# On AMD/HIP, disable cuda graph for DLLM (the attention_backend
# resolution moved to the pipeline: arg_groups/overrides.py
# _dllm_attention_backend, invoked below at its legacy slot).
# DLLM CUDA graphs are disabled on AMD/HIP.
if get_platform().is_hip:
if (
cfg.cuda_graph_config.decode.backend != Backend.DISABLED
@@ -54,7 +53,6 @@ def handle_dllm_inference(server_args: Any):
),
)
run_post_process_pass(server_args, _dllm_attention_backend)
run_post_process_pass(server_args, _dllm_overlap_disable)
# The page-size alignment + block-size cap for dllm moved to the
+11
View File
@@ -1231,6 +1231,11 @@ def _fa4_page_constraint(view: Any) -> dict:
# page_size==1, so skip the 128 auto-force for it and keep the default.
and (view.speculative_eagle_topk or 0) <= 1
):
if (
"DiffusionGemmaForBlockDiffusion"
in model_config_of(view).hf_config.architectures
):
return {"page_size": 1}
logger.warning(
f"FA4 backend only supports page size 128 for non-MLA model architectures, changing page_size from {view.page_size} to 128."
)
@@ -1586,6 +1591,12 @@ def _dllm_attention_backend(view: Any) -> dict:
from sglang.srt.dllm.algorithm import get_algorithm_cls
algorithm_cls = get_algorithm_cls(view.dllm_algorithm)
if view.attention_backend in algorithm_cls.supported_attention_backends and all(
getattr(view, field, None)
in (None, *algorithm_cls.supported_attention_backends)
for field in ("prefill_attention_backend", "decode_attention_backend")
):
return {}
if backend := algorithm_cls.required_attention_backend:
fields = (
"attention_backend",
+6 -5
View File
@@ -34,6 +34,9 @@ class DllmAlgorithm:
supported_architectures: Tuple[str, ...] = ()
requires_separate_context_encoding = False
required_attention_backend: Optional[str] = None
reuse_forward_metadata: bool = False
supported_attention_backends: Tuple[str, ...] = ()
capture_input_preparation: bool = False
def __init__(self, config: DllmConfig):
self.block_size = config.block_size
@@ -113,11 +116,9 @@ class DllmAlgorithm:
if all(start == self.block_size for start in start_list):
return out.logits_output, [], None, None, out.can_run_graph
# NPU: attention metadata is stable across a block's denoise steps (the
# first forward above already planned it), so mark it ready once and let
# every later forward skip re-planning.
if _is_npu:
forward_batch.mark_forward_metadata_ready()
# The first forward has planned the fixed canvas's attention metadata.
if _is_npu or (self.reuse_forward_metadata and out.can_run_graph):
forward_batch.mark_forward_metadata_ready(replan_equivalent=not _is_npu)
for _ in range(self.max_steps(self.block_size)):
done = self.step(forward_batch, out.logits_output.full_logits, states)
if all(done):
@@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Any, List, Optional
import torch
from sglang.srt.arg_groups.model_override_base import model_config_of
from sglang.srt.arg_groups.model_override_base import (
attention_backends_of,
model_config_of,
)
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.dllm.algorithm.base import DllmAlgorithm, DllmRunOutput
@@ -31,11 +34,39 @@ def _denoiser_statistics(logits: torch.Tensor, temperatures: torch.Tensor):
_compiled_denoiser_statistics = torch.compile(_denoiser_statistics, dynamic=True)
def _use_split_denoiser_statistics(logits, temperatures):
return (
logits.dtype == temperatures.dtype == torch.float32
and logits.is_contiguous()
and temperatures.is_contiguous()
and logits.shape[-1] >= 65536
and logits.shape[0] * logits.shape[1] >= 32
and torch.cuda.get_device_capability(logits.device)[0] == 10
)
def _denoiser_statistics_cuda(logits, temperatures, *, write_soft=False):
soft_probabilities = None
if _use_split_denoiser_statistics(logits, temperatures):
from sglang.kernels.ops.sampling.denoiser_statistics import denoiser_statistics
if write_soft:
soft_probabilities = torch.empty_like(logits, dtype=torch.bfloat16)
result = denoiser_statistics(logits, temperatures, soft_probabilities)
else:
result = _compiled_denoiser_statistics(logits, temperatures)
return *result, soft_probabilities
def _sample_denoiser(probabilities: torch.Tensor, generator: torch.Generator):
# The exponential-race formulation samples the same categorical distribution
# as multinomial. Probabilities come from softmax, so multinomial's repeated
# validation and device-to-host synchronization are unnecessary here.
noise = torch.empty_like(probabilities).exponential_(1.0, generator=generator)
if probabilities.is_cuda:
from sglang.kernels.ops.speculative.row_argmax import div_argmax
return div_argmax(probabilities, noise)
return (probabilities / noise).argmax(dim=-1)
@@ -43,6 +74,9 @@ class Gemma4Renoise(DllmAlgorithm):
supported_architectures = ("DiffusionGemmaForBlockDiffusion",)
requires_separate_context_encoding = True
required_attention_backend = "triton"
reuse_forward_metadata = True
supported_attention_backends = ("triton", "fa4")
capture_input_preparation = True
@classmethod
def configure_server_args(cls, server_args: ServerArgs) -> None:
@@ -66,16 +100,24 @@ class Gemma4Renoise(DllmAlgorithm):
):
raise ValueError("DiffusionGemma requires tied word embeddings")
graph_config = cfg.cuda_graph_config
if graph_config.prefill.backend == Backend.FULL:
prefill_backend, _ = attention_backends_of(cfg)
if prefill_backend != "fa4":
raise ValueError(
"Gemma4Renoise FULL prefill CUDA graphs require prefill "
f"backend 'fa4'; got '{prefill_backend}'."
)
else:
graph_config = with_phase(
graph_config, Phase.PREFILL, backend=Backend.DISABLED
)
declare_resolution(
server_args,
cls.__name__,
disable_radix_cache=True,
chunked_prefill_size=-1,
cuda_graph_config=with_phase(
cfg.cuda_graph_config,
Phase.PREFILL,
backend=Backend.DISABLED,
),
cuda_graph_config=graph_config,
)
@classmethod
@@ -153,6 +195,11 @@ class Gemma4Renoise(DllmAlgorithm):
self.seed = algorithm_config.get("seed")
self.vocab_size = None
self.embed_tokens = None
self._temperature_tables = {}
self._temperature_buffers = {}
self._zero_signals = {}
self.use_temperature_table = algorithm_config.get("use_temperature_table", True)
self.use_graph_input_preparation = config.capture_input_preparation
if (
not isinstance(self.max_denoising_steps, int)
@@ -226,40 +273,86 @@ class Gemma4Renoise(DllmAlgorithm):
"argmax": current,
"history": [],
"self_conditioning": None,
"rng_state": generator.get_state(),
"generator": generator,
"finished": False,
}
)
return states
@torch.no_grad()
def prepare_inputs(
self,
model_runner: ModelRunner,
forward_batch: ForwardBatch,
states: List[Any],
) -> None:
self._write_input_ids(forward_batch, states)
if self.fdfo or all(
state["step"] == self.max_denoising_steps for state in states
):
self._write_input_ids(forward_batch, states)
signals = [state["self_conditioning"] for state in states]
signal = next((value for value in signals if value is not None), None)
if signal is not None:
batched = torch.zeros(
(len(states), self.block_size, signal.shape[-1]),
dtype=signal.dtype,
device=signal.device,
if signal is not None or self.use_graph_input_preparation:
weight = self.embed_tokens.weight
key = (weight.device, weight.dtype, weight.shape[-1])
if key not in self._zero_signals:
self._zero_signals[key] = weight.new_zeros(
self.block_size, weight.shape[-1]
)
zero = self._zero_signals[key]
shape = (len(states) * self.block_size, weight.shape[-1])
batched = forward_batch.dllm_input_preparation_state
if batched is None or batched.shape != shape:
batched = weight.new_empty(shape)
torch.stack(
[zero if value is None else value for value in signals],
out=batched.view(len(states), self.block_size, -1),
)
for index, value in enumerate(signals):
if value is not None:
batched[index].copy_(value)
signal = batched.view(len(states) * self.block_size, -1)
forward_batch.dllm_input_preparation_state = batched
signal = batched
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
forward_batch.input_ids, signal
if self.use_graph_input_preparation:
forward_batch.input_embeds = None
else:
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
forward_batch.input_ids, signal
)
@staticmethod
def prepare_graph_inputs(model, input_ids, state):
return model.prepare_dllm_input_embeds(input_ids, state)
def _temperatures(self, logits, states):
if not self.use_temperature_table:
return logits.new_tensor(
[self._temperature(state["step"]) for state in states]
)
key = (logits.device, logits.dtype)
if key not in self._temperature_tables:
self._temperature_tables[key] = logits.new_tensor(
[
self._temperature(step)
for step in range(self.max_denoising_steps + 1)
]
)
table = self._temperature_tables[key]
if len(states) == 1:
step = states[0]["step"]
return table[step : step + 1]
buffer_key = (*key, len(states))
if buffer_key not in self._temperature_buffers:
self._temperature_buffers[buffer_key] = logits.new_empty(len(states))
return torch.stack(
[table[state["step"]] for state in states],
out=self._temperature_buffers[buffer_key],
)
def _write_input_ids(self, forward_batch: ForwardBatch, states: List[Any]) -> None:
current = torch.stack([state["current"] for state in states])
forward_batch.input_ids.copy_(current.view(-1))
torch.stack(
[state["current"] for state in states],
out=forward_batch.input_ids.view(len(states), self.block_size),
)
def _soft_embeddings(self, probabilities: torch.Tensor) -> torch.Tensor:
weight = self.embed_tokens.weight
@@ -292,13 +385,23 @@ class Gemma4Renoise(DllmAlgorithm):
self._write_input_ids(forward_batch, states)
return [True] * len(states)
temperatures = logits.new_tensor(
[self._temperature(state["step"]) for state in states]
)
statistics = (
_compiled_denoiser_statistics if logits.is_cuda else _denoiser_statistics
)
probabilities, token_entropies, argmax_tokens = statistics(logits, temperatures)
temperatures = self._temperatures(logits, states)
soft_probabilities = None
if logits.is_cuda:
probabilities, token_entropies, argmax_tokens, soft_probabilities = (
_denoiser_statistics_cuda(
logits,
temperatures,
write_soft=(
self.embed_tokens.weight.dtype == torch.bfloat16
and getattr(self.embed_tokens, "tp_size", 1) == 1
),
)
)
else:
probabilities, token_entropies, argmax_tokens = _denoiser_statistics(
logits, temperatures
)
sorted_entropy, indices = torch.sort(token_entropies, dim=-1)
cumulative_entropy = torch.cumsum(sorted_entropy, dim=-1)
sorted_selected = cumulative_entropy - sorted_entropy <= self.entropy_bound
@@ -312,8 +415,7 @@ class Gemma4Renoise(DllmAlgorithm):
if state["finished"]:
continue
active.append(index)
generator = torch.Generator(device=logits.device)
generator.set_state(state["rng_state"])
generator = state["generator"]
denoiser = _sample_denoiser(probabilities[index], generator)
random_canvas = torch.randint(
self.vocab_size,
@@ -327,7 +429,8 @@ class Gemma4Renoise(DllmAlgorithm):
if self.stability_threshold == 0:
stable = torch.ones((), dtype=torch.bool, device=logits.device)
elif len(history) == self.stability_threshold:
stable = (torch.stack(history) == argmax).all()
previous = history[0] if len(history) == 1 else torch.stack(history)
stable = (previous == argmax).all()
else:
stable = torch.zeros((), dtype=torch.bool, device=logits.device)
history.append(argmax)
@@ -335,7 +438,6 @@ class Gemma4Renoise(DllmAlgorithm):
history.pop(0)
state["step"] -= 1
state["argmax"] = argmax
state["rng_state"] = generator.get_state()
finished.append((stable & confident[index]) | (state["step"] == 0))
# One device-to-host synchronization for the whole batch. In particular,
@@ -348,7 +450,11 @@ class Gemma4Renoise(DllmAlgorithm):
soft_embeds = None
if not all(done):
soft_embeds = self._soft_embeddings(
probabilities.reshape(-1, self.vocab_size)
(
soft_probabilities
if soft_probabilities is not None
else probabilities
).reshape(-1, self.vocab_size)
).view(len(states), self.block_size, -1)
for index in active:
state = states[index]
+415
View File
@@ -0,0 +1,415 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0.
"""dLLM FA4 compute and graph workspaces over Triton paged-cache metadata."""
from dataclasses import dataclass
from functools import cache
from typing import ClassVar
import torch
from sglang.kernels.ops.attention.dllm_kv_pack import pack_prefix_current
from sglang.kernels.ops.attention.extend_attention import extend_attention_fwd_unified
from sglang.kernels.ops.attention.flash_attention_v4 import flash_attn_gqa_512
from sglang.kernels.ops.attention.flash_attn.cute.interface import (
flash_attn_varlen_func,
)
from sglang.srt.layers.attention.triton_backend import (
ForwardMetadata,
update_sliding_window_buffer,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_capture_attention_variant,
)
from sglang.srt.utils import get_device_capability
class _DenseKVWorkspace:
def __init__(self, capacity, heads, dim, batch_size, device, dtype):
# TMA loads can include the masked tail beyond the packed sequences.
self.key = torch.zeros((capacity, heads, dim), device=device, dtype=dtype)
self.value = torch.zeros_like(self.key)
self.cu_seqlens = torch.empty(batch_size + 1, device=device, dtype=torch.int32)
self.indices = torch.arange(capacity, device=device, dtype=torch.int64)
self.window_start = torch.zeros(batch_size, device=device, dtype=torch.int32)
def fits(self, capacity, batch_size, heads, dim):
return (
capacity * heads * dim <= self.key.numel()
and capacity <= self.indices.numel()
and batch_size < self.cu_seqlens.numel()
)
def view(self, capacity, batch_size, heads, dim):
shape = (capacity, heads, dim)
elements = capacity * heads * dim
return (
self.key.view(-1)[:elements].view(shape),
self.value.view(-1)[:elements].view(shape),
self.cu_seqlens[: batch_size + 1],
self.indices[:capacity],
self.window_start[:batch_size],
)
DLLM_VARLEN = "dllm_varlen"
DLLM_FULL_WINDOW = "dllm_full_window"
@dataclass(frozen=True)
class DllmWindowGraphVariants:
window_size: int
block_size: int
capture_labels: ClassVar[tuple[str, ...]] = (DLLM_VARLEN, DLLM_FULL_WINDOW)
def select(self, forward_batch: ForwardBatch, capture_batch_size: int) -> str:
lengths = forward_batch.seq_lens_cpu
if (
forward_batch.batch_size > 0
and capture_batch_size == forward_batch.batch_size
and forward_batch.forward_mode.is_dllm_extend()
and forward_batch.input_ids.numel()
== forward_batch.batch_size * self.block_size
and lengths is not None
and int(lengths[: forward_batch.batch_size].min()) - self.block_size
>= self.window_size
):
return DLLM_FULL_WINDOW
return DLLM_VARLEN
class DllmFlashAttention:
"""FA4 computation and context-encoding graphs for DiffusionGemma."""
def __init__(self, backend):
assert get_device_capability()[0] == 10, (
"dLLM FA4 D256/D512 requires an SM100-family GPU."
)
self.backend = backend
self.graph_mode = False
self._graph_workspaces = {}
self._eager_workspaces = {}
self._prefill_graph_metadata = {}
self.decode_graph_metadata = {}
self._graph_slots = self._graph_tokens = 0
self._prefill_capture_sizes = set()
self._prefill_capture_max_requests = 0
def init_graph_state(self, max_requests, max_tokens):
self._graph_slots = max(self._graph_slots, max_requests)
self._graph_tokens = max(self._graph_tokens, max_tokens)
def init_prefill_graph_state(self, max_requests, capture_tokens):
self.init_graph_state(max_requests, max(capture_tokens))
self._prefill_capture_sizes = set(capture_tokens)
self._prefill_capture_max_requests = max_requests
def graph_variants(self, block_size):
window = self.backend.sliding_window_size
if window is not None and window > 0:
return DllmWindowGraphVariants(window, block_size)
return None
def _workspace(self, key, capacity, batch_size, heads, dim, query):
if self.graph_mode:
capacity = self._graph_slots * key[0] + self._graph_tokens
batch_size = self._graph_slots
captured = self._graph_workspaces.setdefault(key, [])
candidates = (
captured
if self.graph_mode
else (w for group in self._graph_workspaces.values() for w in group)
)
workspace = next(
(
w
for w in candidates
if w.key.dtype == query.dtype
and w.fits(capacity, batch_size, heads, dim)
),
None,
)
if workspace is None:
workspace = self._eager_workspaces.get(key)
if workspace is None or not workspace.fits(
capacity, batch_size, heads, dim
):
workspace = _DenseKVWorkspace(
capacity, heads, dim, batch_size, query.device, query.dtype
)
if self.graph_mode:
# Keep earlier captures' allocations alive when a later runner needs more capacity.
captured.append(workspace)
self._eager_workspaces.pop(key, None)
else:
self._eager_workspaces[key] = workspace
return workspace
def init_forward_metadata_out_graph(self, batch, in_capture):
self._eager_workspaces.clear()
self.graph_mode = True
if batch.forward_mode == ForwardMode.EXTEND:
return self.init_prefill_metadata(batch, in_capture)
bs = batch.batch_size
if not in_capture:
self.backend.forward_metadata = self.decode_graph_metadata[bs]
self.backend._init_forward_metadata_out_graph(batch, in_capture)
if in_capture:
self.decode_graph_metadata[bs] = self.backend.forward_metadata
def init_prefill_metadata(self, batch, in_capture):
backend = self.backend
bs, tokens = batch.batch_size, batch.input_ids.numel()
key = (bs, tokens)
if in_capture:
self.init_graph_state(bs, tokens)
capacity = batch.max_seq_len_override or backend.max_context_len
swa = (
backend.sliding_window_size is not None
and backend.sliding_window_size > 0
)
self._prefill_graph_metadata[key] = ForwardMetadata(
attn_logits=None,
attn_lse=None,
num_kv_splits=None,
max_extend_len=max(batch.extend_seq_lens_cpu),
kv_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device),
kv_indices=torch.empty(
bs * capacity, dtype=torch.int64, device=backend.device
),
qo_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device),
custom_mask=None,
mask_indptr=None,
window_kv_indptr=torch.zeros(
bs + 1, dtype=torch.int32, device=backend.device
)
if swa
else None,
window_kv_indices=torch.empty(
bs * backend.sliding_window_size,
dtype=torch.int64,
device=backend.device,
)
if swa
else None,
window_num_kv_splits=None,
window_kv_offsets=torch.empty(
bs, dtype=torch.int64, device=backend.device
)
if swa
else None,
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
if backend.use_sliding_window_kv_pool
else None,
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
if backend.kv_index_translator.is_translating
else None,
)
metadata = self._prefill_graph_metadata[key]
backend._fill_kv_indptr_and_indices(
bs,
batch.extend_prefix_lens,
batch.req_pool_indices,
metadata.kv_indices,
kv_indptr=metadata.kv_indptr,
)
metadata.qo_indptr[0].zero_()
torch.cumsum(batch.extend_seq_lens, dim=0, out=metadata.qo_indptr[1:])
if metadata.window_kv_indices is not None:
_, _, _, offsets = update_sliding_window_buffer(
metadata.window_kv_indptr,
backend.kv_index_translator,
batch.req_pool_indices,
backend.sliding_window_size,
batch.extend_prefix_lens,
bs,
backend.device,
backend.token_to_kv_pool,
window_kv_indices=metadata.window_kv_indices,
)
metadata.window_kv_offsets.copy_(offsets)
if metadata.swa_out_cache_loc is not None:
metadata.swa_out_cache_loc.copy_(
backend.kv_index_translator.sliding_window_write_loc_for(
batch.out_cache_loc
)
)
if metadata.out_cache_loc_full_physical is not None:
backend.kv_index_translator.fill_capture_write_loc(
out=metadata.out_cache_loc_full_physical,
forward_batch=batch,
width=metadata.out_cache_loc_full_physical.numel(),
)
backend.forward_metadata = metadata
def can_run_prefill_cuda_graph(self, batch):
query_limit = self.get_prefill_cuda_graph_max_query_len(
batch.input_ids.numel(), self._prefill_capture_max_requests
)
return (
batch.forward_mode == ForwardMode.EXTEND
and batch.input_ids.numel() in self._prefill_capture_sizes
and self.backend.dcp_size == 1
and not batch.contains_image_inputs()
and (
query_limit is None
or max(batch.extend_seq_lens_cpu, default=0) <= query_limit
)
)
def get_prefill_cuda_graph_max_query_len(self, num_tokens, max_requests):
if num_tokens <= max_requests * self.backend.dllm_block_size:
return self.backend.dllm_block_size
return None
@cache
def _supports_layer(self, layer, dtype):
pool = self.backend.token_to_kv_pool
buffers = (
pool.get_key_buffer(layer.layer_id),
pool.get_value_buffer(layer.layer_id),
)
return (
layer.qk_head_dim == layer.v_head_dim
and layer.qk_head_dim in (256, 512)
and dtype in (torch.float16, torch.bfloat16)
and all(
buffer.ndim == 3 and buffer.dtype == dtype and buffer.stride(-1) == 1
for buffer in buffers
)
)
def forward_extend(
self,
layer,
q,
k,
v,
out,
kb,
vb,
qo,
ki,
ids,
mask,
causal,
mask_ptr,
max_q,
k_scale,
v_scale,
sm_scale=None,
**options,
):
if (
not self._supports_layer(layer, q.dtype)
or mask is not None
or options.get("sinks") is not None
or options.get("score_mod") is not None
or options.get("logit_cap", 0.0)
):
return self.backend.extend_attention_fwd(
q,
k.contiguous(),
v.contiguous(),
out,
kb,
vb,
qo,
ki,
ids,
mask,
causal,
mask_ptr,
max_q,
k_scale,
v_scale,
sm_scale=sm_scale,
**options,
)
bs, heads, dim = qo.numel() - 1, k.shape[1], k.shape[2]
is_window_layer = (
layer.sliding_window_size is not None and layer.sliding_window_size >= 0
)
prefix_capacity = (
self.backend.sliding_window_size
if is_window_layer
else self.backend.max_context_len
)
prefix_tokens = bs * prefix_capacity if self.graph_mode else ids.numel()
capacity = prefix_tokens + q.shape[0]
key = (prefix_capacity, heads, dim, q.dtype)
workspace = self._workspace(key, capacity, bs, heads, dim, q)
dense_k, dense_v, cu_seqlens, dense_ids, window_start = workspace.view(
capacity, bs, heads, dim
)
pack_prefix_current(
k,
v,
kb,
vb,
qo,
ki,
ids,
dense_k,
dense_v,
cu_seqlens,
prefix_capacity + max_q,
)
window = options.get("sliding_window_size", -1)
if window > 0:
prefix_lens = ki[1:] - ki[:-1]
return extend_attention_fwd_unified(
q,
out,
dense_k,
dense_v,
k_scale,
v_scale,
qo,
cu_seqlens,
dense_ids,
prefix_lens,
max_q,
sm_scale=sm_scale,
is_causal=causal,
sliding_window_size=window,
window_start_pos=window_start,
page_size=1,
)
if dim == 512:
return flash_attn_gqa_512(
q,
dense_k,
dense_v,
out,
cu_seqlens_q=qo,
cu_seqlens_k=cu_seqlens,
softmax_scale=sm_scale,
causal=causal,
)
if (
is_window_layer
and get_capture_attention_variant() == DLLM_FULL_WINDOW
and not causal
):
return flash_attn_varlen_func(
q.view(bs, -1, *q.shape[1:]),
dense_k.view(bs, -1, heads, dim),
dense_v.view(bs, -1, heads, dim),
out=out.view(bs, -1, *out.shape[1:]),
softmax_scale=sm_scale,
causal=False,
)
return flash_attn_varlen_func(
q,
dense_k,
dense_v,
out=out,
cu_seqlens_q=qo,
cu_seqlens_k=cu_seqlens,
max_seqlen_q=max_q,
max_seqlen_k=prefix_capacity + max_q,
softmax_scale=sm_scale,
causal=causal,
)
+6
View File
@@ -15,6 +15,7 @@ class DllmConfig:
max_running_requests: int,
first_done_first_out_mode: bool = False,
requires_separate_context_encoding: bool = False,
capture_input_preparation: bool = False,
):
self.algorithm = algorithm
self.algorithm_config = algorithm_config
@@ -23,6 +24,7 @@ class DllmConfig:
self.max_running_requests = max_running_requests
self.first_done_first_out_mode = first_done_first_out_mode
self.requires_separate_context_encoding = requires_separate_context_encoding
self.capture_input_preparation = capture_input_preparation
def validate_request(self, req) -> str | None:
from sglang.srt.dllm.algorithm import get_algorithm_cls
@@ -113,4 +115,8 @@ class DllmConfig:
requires_separate_context_encoding=(
algorithm_cls.requires_separate_context_encoding
),
capture_input_preparation=(
algorithm_cls.capture_input_preparation
and algorithm_config.get("capture_input_preparation", True)
),
)
@@ -253,6 +253,10 @@ def create_flashattention_v3_backend(runner):
@register_attention_backend("fa4")
def create_flashattention_v4_backend(runner):
if "DiffusionGemmaForBlockDiffusion" in runner.model_config.hf_config.architectures:
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
return TritonAttnBackend(runner, dllm_fa4=True)
from sglang.srt.layers.attention.flashattention_backend import (
FlashAttentionBackend,
)
@@ -159,6 +159,7 @@ class AttentionBackend(ABC):
# True when prefill graph metadata can use ForwardBatch.max_seq_len_override.
supports_prefill_cuda_graph_max_context_size: bool = False
dllm_attention = None
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
"""Declare where this backend's scheduler-shared reads end per mode.
@@ -13,6 +13,7 @@ from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.runtime_context import (
get_exec,
get_spec,
)
@@ -93,6 +94,10 @@ class HybridAttnBackend(AttentionBackend):
def supports_full_cuda_graph_chunked_prefix(self) -> bool:
return self.prefill_backend.supports_full_cuda_graph_chunked_prefix
@property
def dllm_attention(self):
return self.prefill_backend.dllm_attention
@property
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
return self.prefill_backend.supports_prefill_cuda_graph_max_context_size
@@ -125,12 +130,15 @@ class HybridAttnBackend(AttentionBackend):
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
self.decode_backend.init_cuda_graph_state(max_bs, max_num_tokens)
if get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill:
# When speculative decoding is enabled, we need to initialize the backend
# that will be used for target_verify.
if get_exec().dllm.dllm_algorithm is not None or (
get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill
):
# DLLM and prefill-backed verification capture the prefill backend.
self.prefill_backend.init_cuda_graph_state(max_bs, max_num_tokens)
def get_cuda_graph_seq_len_fill_value(self):
if get_exec().dllm.dllm_algorithm is not None:
return self.prefill_backend.get_cuda_graph_seq_len_fill_value()
return self.decode_backend.get_cuda_graph_seq_len_fill_value()
def init_mha_chunk_metadata(
@@ -159,35 +167,12 @@ class HybridAttnBackend(AttentionBackend):
).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs)
def validate_elastic_cuda_graph_recapture(self) -> None:
self.decode_backend.validate_elastic_cuda_graph_recapture()
def forward(
self,
q: Optional[torch.Tensor] = None, # For full attention
k: Optional[torch.Tensor] = None, # For full attention
v: Optional[torch.Tensor] = None, # For full attention
layer: Optional[RadixAttention] = None,
forward_batch: Optional[ForwardBatch] = None,
save_kv_cache: bool = True,
*,
mixed_qkv: Optional[torch.Tensor] = None, # For linear attention
a: Optional[torch.Tensor] = None, # For linear attention
b: Optional[torch.Tensor] = None, # For linear attention
**kwargs,
):
"""Forward method that supports both regular attention (q, k, v) and linear attention (mixed_qkv, a, b)."""
backend = self._select_backend(forward_batch.forward_mode)
if mixed_qkv is not None:
return backend.forward(
layer=layer,
forward_batch=forward_batch,
save_kv_cache=save_kv_cache,
mixed_qkv=mixed_qkv,
a=a,
b=b,
**kwargs,
)
return backend.forward(q, k, v, layer, forward_batch, save_kv_cache, **kwargs)
backend = (
self.prefill_backend
if get_exec().dllm.dllm_algorithm is not None
else self.decode_backend
)
backend.validate_elastic_cuda_graph_recapture()
def forward_decode(
self,
@@ -32,6 +32,10 @@ class TboAttnBackend(AttentionBackend):
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
return self.primary.supports_prefill_cuda_graph_max_context_size
@property
def dllm_attention(self):
return self.primary.dllm_attention
@classmethod
def init_new(cls, creator: Callable[[], AttentionBackend]):
return cls(
@@ -154,6 +154,8 @@ class TritonAttnBackend(AttentionBackend):
model_runner: ModelRunner,
skip_prefill: bool = False,
kv_indptr_buf: Optional[torch.Tensor] = None,
*,
dllm_fa4: bool = False,
):
# Lazy import to avoid the initialization of cuda context
from sglang.kernels.ops.attention.decode_attention import (
@@ -385,7 +387,9 @@ class TritonAttnBackend(AttentionBackend):
if not self.skip_prefill:
self.qo_indptr = torch.zeros(
(max_bs + 1,), dtype=torch.int64, device=model_runner.device
(max_bs + 1,),
dtype=torch.int32 if dllm_fa4 else torch.int64,
device=model_runner.device,
)
self.mask_indptr = torch.zeros(
@@ -409,6 +413,11 @@ class TritonAttnBackend(AttentionBackend):
Lq=head_dim, Lv=head_dim
)
self.extend_attention_block_m = block_m
if dllm_fa4:
from sglang.srt.dllm.attention import DllmFlashAttention
self.dllm_attention = DllmFlashAttention(self)
self.supports_prefill_cuda_graph_max_context_size = True
def get_num_kv_splits(
self,
@@ -498,8 +507,10 @@ class TritonAttnBackend(AttentionBackend):
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
kv_indices: torch.Tensor,
kv_indptr: Optional[torch.Tensor] = None,
) -> torch.Tensor:
kv_indptr = self.kv_indptr[: bs + 1]
if kv_indptr is None:
kv_indptr = self.kv_indptr[: bs + 1]
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
self.kv_index_translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
@@ -703,6 +714,13 @@ class TritonAttnBackend(AttentionBackend):
forward_batch: ForwardBatch,
in_capture: bool = False,
):
if self.dllm_attention is not None:
return self.dllm_attention.init_forward_metadata_out_graph(
forward_batch, in_capture
)
self._init_forward_metadata_out_graph(forward_batch, in_capture)
def _init_forward_metadata_out_graph(self, forward_batch, in_capture):
bs = forward_batch.batch_size
req_pool_indices = forward_batch.req_pool_indices
seq_lens = forward_batch.seq_lens
@@ -810,6 +828,8 @@ class TritonAttnBackend(AttentionBackend):
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Init auxiliary variables for triton attention backend."""
if self.dllm_attention is not None:
self.dllm_attention.graph_mode = False
self._dense_one_shot_kv_indptr = None
bs = forward_batch.batch_size
window_kv_indptr = self.window_kv_indptr
@@ -1072,6 +1092,8 @@ class TritonAttnBackend(AttentionBackend):
kv_indices_buf: Optional[torch.Tensor] = None,
cuda_graph_num_kv_splits_buf: Optional[torch.Tensor] = None,
):
if self.dllm_attention is not None:
self.dllm_attention.init_graph_state(max_bs, max_num_tokens)
self.cuda_graph_attn_logits = torch.zeros(
(max_num_tokens, self.num_head, self.max_kv_splits, self.v_head_dim),
dtype=torch.float32,
@@ -1789,10 +1811,11 @@ class TritonAttnBackend(AttentionBackend):
):
return o
self.extend_attention_fwd(
self._forward_extend_kernel(
layer,
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
k.contiguous(),
v.contiguous(),
k.contiguous() if self.dllm_attention is None else k,
v.contiguous() if self.dllm_attention is None else v,
o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
self.token_to_kv_pool.get_key_buffer(layer.layer_id),
self.token_to_kv_pool.get_value_buffer(layer.layer_id),
@@ -1818,6 +1841,11 @@ class TritonAttnBackend(AttentionBackend):
)
return o
def _forward_extend_kernel(self, layer, *args, **kwargs):
if self.dllm_attention is not None:
return self.dllm_attention.forward_extend(layer, *args, **kwargs)
return self.extend_attention_fwd(*args, **kwargs)
def _dense_one_shot_kv_indptr_for(self, forward_batch: ForwardBatch):
"""Cumulative full sequence lengths addressing the one-shot K/V rows.
@@ -2589,6 +2617,11 @@ def update_sliding_window_buffer(
window_kv_indptr[-1], dtype=torch.int64, device=device
)
window_kv_start_idx = seq_lens - window_kv_lens
token_mapping = (
token_to_kv_pool.full_to_swa_index_mapping
if isinstance(token_to_kv_pool, SWAKVPool)
else None
)
translated = translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
seq_lens=window_kv_lens,
@@ -2597,6 +2630,7 @@ def update_sliding_window_buffer(
out=window_kv_indices,
kv_start_idx=window_kv_start_idx,
sliding_window=translator.reads_are_translated,
token_mapping=token_mapping,
)
if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool):
kv_last_index = window_kv_indptr[-1]
+22 -6
View File
@@ -26,14 +26,14 @@ from torch import nn
from sglang.kernels.ops.activation.softcap import (
softcap_inplace_logits as fused_softcap,
)
from sglang.kernels.ops.activation.softcap import (
softcap_to_float32_logits,
)
from sglang.srt.beam_search.logits_capture import BeamLogitsCapture
from sglang.srt.distributed.device_communicators import triton_symm_mem_ag
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStates,
pack_aux_hidden_states,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStates, pack_aux_hidden_states
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
attn_tp_all_gather,
@@ -945,11 +945,19 @@ class LogitsProcessor(nn.Module):
"dp_logits_scatter_returned", logits_shape=tuple(logits.shape)
)
fuse_cast_softcap = (
self.final_logit_softcapping
and logits.is_cuda
and logits.dtype in (torch.float16, torch.bfloat16)
)
logits = self._copy_logits_to_buffer(
logits, logits_metadata, use_buffer=use_logits_buffer
logits,
logits_metadata,
use_buffer=use_logits_buffer,
softcap=self.final_logit_softcapping if fuse_cast_softcap else None,
)
if self.final_logit_softcapping:
if self.final_logit_softcapping and not fuse_cast_softcap:
if not (_is_npu or _is_cpu):
fused_softcap(logits, self.final_logit_softcapping)
else:
@@ -1148,11 +1156,19 @@ class LogitsProcessor(nn.Module):
logits: torch.Tensor,
logits_metadata: LogitsMetadata,
use_buffer: bool = True,
softcap: Optional[float] = None,
) -> torch.Tensor:
logits_buffer = logits_metadata.next_token_logits_buffer if use_buffer else None
if logits.shape[-1] > self.vocab_size:
logits = logits[:, : self.vocab_size]
logits_width = logits.shape[-1]
if softcap is not None:
if logits_buffer is None or tuple(logits_buffer.shape) != tuple(
logits.shape
):
logits_buffer = torch.empty_like(logits, dtype=torch.float32)
assert logits_buffer.dtype == torch.float
return softcap_to_float32_logits(logits, softcap, logits_buffer)
# The shared logits buffer is keyed by vocab width and rows; skip it
# when this batch has a different logits shape than the graph buffer.
if logits_buffer is not None and tuple(logits_buffer.shape) == tuple(
@@ -0,0 +1,163 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1
},
"128": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1
},
"129": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"255": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3
},
"257": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"510": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"511": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"513": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"514": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"1022": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"1023": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"1025": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"1026": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"2046": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"2047": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2049": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2050": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"8192": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
}
}
@@ -0,0 +1,164 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1
},
"128": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1
},
"129": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"255": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"257": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"510": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"511": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"513": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"514": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"1022": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"1023": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"1025": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": false
},
"1026": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"2046": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"2047": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2049": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"USE_TMA": true
},
"2050": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
},
"8192": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8
}
}
@@ -194,11 +194,12 @@ class KVIndexTranslator:
out: torch.Tensor,
kv_start_idx: Optional[torch.Tensor] = None,
sliding_window: bool = False,
token_mapping: Optional[torch.Tensor] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
report whether they came out translated.
Non-unified: the historical gather straight from ``req_to_token``.
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
Unified: one fused gather-and-translate, so no caller needs a
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
resident token, a length the pool bounds.
@@ -206,8 +207,8 @@ class KVIndexTranslator:
``sliding_window`` selects the swa sub-pool's own id space, built from
VIRTUAL ids and never chained through full-physical. A ``False`` return
means the ids are still VIRTUAL: the DCP path defers translation to
``translate_dcp_read_ids``, and a static SWA pool maps the full ids
through its own full->swa table.
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
table as ``token_mapping`` to fuse that translation into the gather.
"""
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
# (the padded graph buffer), and the extra lanes have no length to bound.
@@ -228,8 +229,9 @@ class KVIndexTranslator:
out,
self.req_to_token.stride(0),
ENTRY_PAGE_SIZE=1,
token_mapping=token_mapping,
)
return False
return token_mapping is not None
if sliding_window:
assert self._swa_v2p_table is not None, (
@@ -556,6 +556,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# For input embeddings
input_embeds: Optional[torch.Tensor] = None
dllm_input_preparation_state: Optional[torch.Tensor] = None
# For token embedding overrides (sparse replacement at specific positions)
replace_embeds: Optional[torch.Tensor] = None
replace_positions: Optional[torch.Tensor] = None
@@ -39,9 +39,7 @@ from torch.profiler import ProfilerActivity, profile
from sglang.srt.compilation import torch_compile_decoration
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
from sglang.srt.distributed.parallel_state import (
graph_capture,
)
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.environ import envs
from sglang.srt.layers.attention.base_attn_backend import (
@@ -63,6 +61,8 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.model_executor.cuda_graph_buffer_registry import (
CudaGraphBufferRegistry,
GraphSlot,
PaddingPolicy,
build_decode_registry,
)
from sglang.srt.model_executor.forward_batch_info import (
@@ -89,12 +89,8 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor
BreakableCudaGraphBackend,
)
from sglang.srt.model_executor.runner_backend.utils import resolve_decode_backend
from sglang.srt.model_executor.runner_backend_utils import (
CUDA_GRAPH_CAPTURE_FAILED_MSG,
)
from sglang.srt.model_executor.runner_utils.buffers import (
DecodeInputBuffers,
)
from sglang.srt.model_executor.runner_backend_utils import CUDA_GRAPH_CAPTURE_FAILED_MSG
from sglang.srt.model_executor.runner_utils.buffers import DecodeInputBuffers
from sglang.srt.model_executor.runner_utils.capture_mode import (
_set_capture_attention_variant,
_set_capture_lora_variant,
@@ -108,12 +104,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
)
from sglang.srt.model_executor.runner_utils.shared_read_event import make_external_event
from sglang.srt.multiplex.pdmux_context import get_current_stream_idx, get_stream_groups
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_parallel,
get_spec,
)
from sglang.srt.runtime_context import get_exec, get_flags, get_parallel, get_spec
from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
from sglang.srt.utils import (
empty_context,
@@ -230,6 +221,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
pluggable self.backend that handles the actual capture/replay.
"""
dllm_input_preparation = None
dllm_attention = None
def __init__(
self,
model_runner: ModelRunner,
@@ -278,6 +272,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.dllm_config = DllmConfig.from_server_args(model_runner.server_args)
self.is_dllm = self.dllm_config is not None
self.dllm_input_preparation = None
if self.is_dllm and self.dllm_config.capture_input_preparation:
from sglang.srt.dllm.algorithm import get_algorithm_cls
self.dllm_input_preparation = get_algorithm_cls(
self.dllm_config.algorithm
).prepare_graph_inputs
self.dllm_uses_input_embeds = (
self.is_dllm and self.dllm_config.requires_separate_context_encoding
)
@@ -309,13 +310,22 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
elif self.is_dllm:
self.capture_forward_mode = ForwardMode.DLLM_EXTEND
self.dllm_attention = self.attn_backend.dllm_attention
self.attention_graph_variants: Optional[AttentionGraphVariants] = (
create_attention_graph_variants(model_runner.model_config.hf_config)
self.dllm_attention.graph_variants(self.captured_req_width)
if self.dllm_attention is not None and self.is_dllm
else create_attention_graph_variants(model_runner.model_config.hf_config)
or create_dsv41_candidate_graph_variants(
model_runner, self.capture_forward_mode, self.captured_req_width
)
)
if (
self.dllm_attention is not None
and self.attention_graph_variants is not None
):
self._resolve_attention_variant = self.attention_graph_variants.select
# --- bucket sizes ---------------------------------------------
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(
model_runner, self.captured_req_width
@@ -461,6 +471,17 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
num_dp_ranks=self.num_dp_ranks,
source=self.buffers,
)
if self.dllm_input_preparation is not None:
width = self.buffers.input_embeds.shape[-1]
self.buffer_registry.register_slot(
GraphSlot(
"dllm_input_preparation_state",
lambda bs, tokens: (tokens, width),
self.buffers.input_embeds.dtype,
axis="tokens",
padding_policy=PaddingPolicy.ZERO,
)
)
# Captures the per-replay attention-metadata prep into a small CUDA
# graph; see metadata_glue_graph.py for the correctness contract.
@@ -589,9 +610,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
return [num_tokens]
return None
def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
def _resolve_attention_variant(
self, forward_batch: ForwardBatch, capture_batch_size: int
) -> Optional[str]:
variants = self.attention_graph_variants
return variants.select(forward_batch) if variants is not None else None
if variants is None:
return None
return variants.select(forward_batch)
def _resolve_lora_variant(self, forward_batch: ForwardBatch):
if not self.record_nolora_graph:
@@ -705,7 +730,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
stream_idx=get_current_stream_idx() if self.enable_pdmux else None,
variant_label=self._resolve_lora_variant(forward_batch),
attention_variant=(
self._resolve_attention_variant(forward_batch)
self._resolve_attention_variant(forward_batch, cuda_graph_bs)
if self.disable_padding
else None
),
@@ -1114,10 +1139,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
else [(None, None)]
)
variants = self.attention_graph_variants
attention_variants = (
variants.capture_labels if variants is not None else (None,)
)
for bs in capture_range:
attention_variants = (
variants.capture_labels if variants is not None else (None,)
)
if get_parallel().tp_rank == 0:
avail_mem = get_available_gpu_memory(
self.model_runner.device,
@@ -1220,7 +1245,15 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
if self.dllm_input_preparation is None:
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
else:
state = self.buffer_registry.get_slot(
"dllm_input_preparation_state"
).buffer[:num_tokens]
kwargs["input_embeds"] = self.dllm_input_preparation(
self.model_runner.model, forward_batch.input_ids, state
)
out = forward(
forward_batch.input_ids,
@@ -1280,7 +1313,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch: ForwardBatch,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
):
if self.dllm_uses_input_embeds and forward_batch.input_embeds is None:
if (
self.dllm_uses_input_embeds
and self.dllm_input_preparation is None
and forward_batch.input_embeds is None
):
raise ValueError(
"Diffusion graph replay requires prepared input embeddings"
)
@@ -1294,6 +1331,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.deepep_adapter.replay()
if not forward_batch.needs_forward_metadata_init():
if self.dllm_input_preparation is not None:
slot = self.buffer_registry.get_slot("dllm_input_preparation_state")
source = forward_batch.dllm_input_preparation_state
padded_tokens = self.bs * self.captured_req_width
slot.buffer[: self.raw_num_token].copy_(source)
slot.reset_padding(self.raw_num_token, padded_tokens)
self.buffers.input_ids[self.raw_num_token : padded_tokens].zero_()
# Pre-planned (plan-stream load_batch already ran).
# In speculative decoding, these two fields are still needed.
graph_size_key = (
@@ -1339,7 +1383,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch.input_embeds
)
variant_label = self._resolve_lora_variant(forward_batch)
attention_variant = self._resolve_attention_variant(forward_batch)
attention_variant = self._resolve_attention_variant(forward_batch, self.bs)
stream_idx = get_current_stream_idx() if self.enable_pdmux else None
self._replay_graph_key = self._make_graph_key(
graph_size_key, stream_idx, variant_label, attention_variant
@@ -1472,7 +1516,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.model_runner.hisparse_coordinator.num_real_reqs.fill_(raw_bs)
variant_label = self._resolve_lora_variant(forward_batch)
attention_variant = self._resolve_attention_variant(forward_batch)
attention_variant = self._resolve_attention_variant(forward_batch, bs)
stream_idx = get_current_stream_idx() if self.enable_pdmux else None
self._replay_graph_key = self._make_graph_key(
graph_size_key, stream_idx, variant_label, attention_variant
@@ -288,6 +288,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
_use_draft_input_embeds = False
_backend_can_run_prefill_cuda_graph = None
dllm_attention = None
def __init__(self, model_runner: ModelRunner):
if get_schedule().enable_mixed_chunk:
@@ -297,8 +298,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
f"graph backend; got '{backend}'."
)
super().__init__(model_runner)
self._backend_can_run_prefill_cuda_graph = getattr(
model_runner.attn_backend, "can_run_prefill_cuda_graph", None
self.dllm_attention = model_runner.attn_backend.dllm_attention
self._backend_can_run_prefill_cuda_graph = (
self.dllm_attention.can_run_prefill_cuda_graph
if self.dllm_attention is not None
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
)
# --- model flags ----------------------------------------------
self.quant_config = getattr(model_runner.model, "quant_config", None)
@@ -481,6 +485,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
max_req = prefill_config.full_prefill_max_req
assert max_req is not None, "full_prefill_max_req must be resolved"
self._capture_req_slots = max_req
if self.dllm_attention is not None:
self.dllm_attention.init_prefill_graph_state(
max_req, self.capture_num_tokens
)
# BCG/Full record LoRA kernels, so the metadata they read must live in
# static buffers refreshed in place per batch; unsupported LoRA
@@ -505,7 +513,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# This flag controls whether the model dispatches through the distinct
# chunked-prefix topology; backend capability is validated separately.
self._capture_chunked_prefix = (
self._is_full_backend and not get_schedule().disable_chunked_prefix_cache
self._is_full_backend
and not get_schedule().disable_chunked_prefix_cache
and self.dllm_attention is None
)
self._prefix_chunk_len = 0
self._prefix_chunk_capacity = 0
@@ -1406,6 +1416,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
context_length = min(
self.max_context_size or model_context_length, model_context_length
)
if self.dllm_attention is not None and self._is_full_backend:
query_limit = self.dllm_attention.get_prefill_cuda_graph_max_query_len(
num_tokens, self._capture_req_slots
)
if query_limit is not None:
context_length = min(context_length, query_limit)
# A prefill bucket is an aggregate token count. Capture it as the
# fewest synthetic requests, with every request containing no more
# than context_length tokens.
@@ -2153,7 +2169,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
def _finalize_execute_output(
self, output
) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]:
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
if output is None:
return None
if isinstance(output, LogitsProcessorOutput):
return self._trim_logits_output(output)
if isinstance(output, EmbeddingPoolerOutput):
@@ -2170,7 +2188,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
def execute(
self, forward_batch: ForwardBatch, **kwargs
) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]:
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
self._validate_capture_hidden_mode(forward_batch)
with self.backend.replay_session():
static_forward_batch = self.load_batch(forward_batch, **kwargs)
+57 -8
View File
@@ -21,7 +21,13 @@ import torch
from torch import nn
from transformers import PreTrainedModel
from sglang.kernels.ops.attention.gemma_qkv_norm_rope import gemma_qkv_norm_rope
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
from sglang.kernels.ops.layernorm.rmsnorm_fanout import (
can_use_rmsnorm_fanout,
rmsnorm_fanout,
)
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.activation import GeluAndMul
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
from sglang.srt.layers.linear import (
@@ -57,7 +63,7 @@ from sglang.srt.models.gemma4_mm import (
)
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils import add_prefix, is_sm100_supported, make_pp_layers
logger = logging.getLogger(__name__)
@@ -162,7 +168,28 @@ class DiffusionGemmaAttention(nn.Module):
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if q.is_cuda:
fuse_rope = (
q.is_cuda
and q.dtype in (torch.bfloat16, torch.float16)
and 0 < q.shape[0] <= 1024
and is_sm100_supported()
)
if fuse_rope:
gemma_qkv_norm_rope(
q,
k,
v,
self.q_norm.weight,
self.k_norm.weight,
self.rotary_emb.cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rotary_emb.rotary_dim,
self.q_norm.eps,
)
elif q.is_cuda:
gemma_qkv_rmsnorm(
q,
k,
@@ -185,7 +212,8 @@ class DiffusionGemmaAttention(nn.Module):
v.unflatten(-1, (self.num_kv_heads, self.head_dim))
).flatten(-2, -1)
q, k = self.rotary_emb(positions, q, k)
if not fuse_rope:
q, k = self.rotary_emb(positions, q, k)
attn = (
self.attn
@@ -289,6 +317,7 @@ class DiffusionGemmaDecoderLayer(nn.Module):
),
)
self.use_fanout = is_sm100_supported() and not is_batch_invariant_mode_enabled()
eps = config.rms_norm_eps
self.input_layernorm = RMSNorm(config.hidden_size, eps=eps)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=eps)
@@ -310,14 +339,30 @@ class DiffusionGemmaDecoderLayer(nn.Module):
hidden_states = residual + hidden_states
residual = hidden_states
h1 = self.post_feedforward_layernorm_1(
self.mlp(self.pre_feedforward_layernorm(residual))
)
use_fanout = self.use_fanout and can_use_rmsnorm_fanout(residual)
if use_fanout:
if not self.router._scale_fused:
self.router.fuse_scale()
dense_in, moe_in, router_in = rmsnorm_fanout(
residual,
self.pre_feedforward_layernorm.weight,
self.pre_feedforward_layernorm_2.weight,
self.router.norm.weight,
self.pre_feedforward_layernorm.variance_epsilon,
)
else:
dense_in = self.pre_feedforward_layernorm(residual)
h1 = self.post_feedforward_layernorm_1(self.mlp(dense_in))
del dense_in
if use_fanout:
router_logits, _ = self.router.proj(router_in)
del router_in
else:
moe_in = self.pre_feedforward_layernorm_2(residual)
router_logits = self.router(residual)
# MoE branch: the router gates on the RAW residual (the router's input
# norm is with_scale=False, so ln2's learned per-channel weights would
# change the routed direction). Experts still get the ln2-normed input.
moe_in = self.pre_feedforward_layernorm_2(residual)
router_logits = self.router(residual)
h2 = self.post_feedforward_layernorm_2(self.moe(moe_in, router_logits))
hidden_states = self.post_feedforward_layernorm(h1 + h2)
@@ -479,6 +524,10 @@ class DiffusionGemmaForBlockDiffusion(PreTrainedModel):
**kwargs,
):
is_context_encoding = not forward_batch.forward_mode.is_dllm_extend()
if not is_context_encoding and input_embeds is None:
input_embeds = self.prepare_dllm_input_embeds(
input_ids, forward_batch.dllm_input_preparation_state
)
if is_context_encoding and forward_batch.contains_image_inputs():
Gemma4ForConditionalGeneration.prepare_attn_masks(
self, forward_batch, input_ids, torch.bool