[Perf] Accelerate DiffusionGemma with FA4 and CUDA graphs on Blackwell (#42239)
Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
lianghougong
Xiaoyu Zhang
parent
2f6e0ec45d
commit
d7190b0122
@@ -344,6 +344,20 @@ for _fn in ("softcap_out", "softcap_inplace_logits"):
|
||||
del _fn
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="activation.softcap_to_float32_logits",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits",
|
||||
capabilities=_CUDA,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16", "float32"),
|
||||
description="Softcap logits into a caller-provided FP32 output.",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
|
||||
@@ -75,6 +75,9 @@ def softcap_inplace_logits_kernel(
|
||||
ncols,
|
||||
row_stride,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
output_ptr=None,
|
||||
output_row_stride=0,
|
||||
CAST_TO_FP32: tl.constexpr = False,
|
||||
):
|
||||
row = tl.program_id(1).to(tl.int64)
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
@@ -85,18 +88,24 @@ def softcap_inplace_logits_kernel(
|
||||
# Load values
|
||||
row_ptr = full_logits_ptr + row * row_stride
|
||||
x = tl.load(row_ptr + offsets, mask=mask)
|
||||
if CAST_TO_FP32:
|
||||
x = x.to(tl.float32)
|
||||
|
||||
# Perform operations in-place
|
||||
x = x / softcapping_value
|
||||
x = libdevice.tanh(x)
|
||||
x = x * softcapping_value
|
||||
|
||||
# Store result
|
||||
tl.store(row_ptr + offsets, x, mask=mask)
|
||||
if CAST_TO_FP32:
|
||||
tl.store(output_ptr + row * output_row_stride + offsets, x, mask=mask)
|
||||
else:
|
||||
tl.store(row_ptr + offsets, x, mask=mask)
|
||||
|
||||
|
||||
def softcap_inplace_logits(full_logits, final_logit_softcapping):
|
||||
if full_logits.is_contiguous():
|
||||
def _softcap_logits(full_logits, final_logit_softcapping, output=None):
|
||||
if output is not None:
|
||||
assert output.shape == full_logits.shape and output.dtype == torch.float32
|
||||
if full_logits.is_contiguous() and (output is None or output.is_contiguous()):
|
||||
nrows, ncols = 1, full_logits.numel()
|
||||
row_stride = ncols
|
||||
else:
|
||||
@@ -106,6 +115,10 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping):
|
||||
)
|
||||
nrows, ncols = full_logits.shape
|
||||
row_stride = full_logits.stride(0)
|
||||
if output is not None:
|
||||
assert output.stride(1) == 1 and (
|
||||
nrows <= 1 or output.stride(0) >= ncols
|
||||
), "softcap output requires contiguous, non-overlapping rows"
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = ((ncols + BLOCK_SIZE - 1) // BLOCK_SIZE, nrows)
|
||||
@@ -116,5 +129,19 @@ def softcap_inplace_logits(full_logits, final_logit_softcapping):
|
||||
ncols=ncols,
|
||||
row_stride=row_stride,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
output_ptr=output,
|
||||
output_row_stride=(ncols if nrows == 1 else output.stride(0))
|
||||
if output is not None
|
||||
else 0,
|
||||
CAST_TO_FP32=output is not None,
|
||||
)
|
||||
return full_logits
|
||||
return full_logits if output is None else output
|
||||
|
||||
|
||||
def softcap_inplace_logits(full_logits, final_logit_softcapping):
|
||||
return _softcap_logits(full_logits, final_logit_softcapping)
|
||||
|
||||
|
||||
def softcap_to_float32_logits(logits, final_logit_softcapping, output):
|
||||
"""Write softcapped FP32 logits to an equally shaped, row-contiguous output."""
|
||||
return _softcap_logits(logits, final_logit_softcapping, output)
|
||||
|
||||
@@ -360,6 +360,43 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.pack_prefix_current",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
description="Pack CSR prefix and current K/V into caller-provided dense buffers.",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.flash_attn_gqa_512",
|
||||
backend=KernelBackend.CUTE_DSL,
|
||||
target="sglang.kernels.ops.attention.flash_attention_v4:flash_attn_gqa_512",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))}
|
||||
),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16"),
|
||||
description="SM100/SM110 D512 GQA with separate K/V and optional page tables.",
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.gemma_qkv_norm_rope",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16"),
|
||||
in_place=True,
|
||||
description="Strided QKV normalization and partial NeoX RoPE.",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
"""Pack a CSR prefix and current K/V into capture-stable dense storage."""
|
||||
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_kv(
|
||||
K,
|
||||
V,
|
||||
KB,
|
||||
VB,
|
||||
QO,
|
||||
KI,
|
||||
IDS,
|
||||
KD,
|
||||
VD,
|
||||
CU,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
KS: tl.constexpr,
|
||||
KH: tl.constexpr,
|
||||
VS: tl.constexpr,
|
||||
VH: tl.constexpr,
|
||||
KBS: tl.constexpr,
|
||||
KBH: tl.constexpr,
|
||||
VBS: tl.constexpr,
|
||||
VBH: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
qs = tl.load(QO + row).to(tl.int64)
|
||||
qe = tl.load(QO + row + 1).to(tl.int64)
|
||||
ps = tl.load(KI + row).to(tl.int64)
|
||||
pe = tl.load(KI + row + 1).to(tl.int64)
|
||||
prefix = pe - ps
|
||||
length = prefix + qe - qs
|
||||
start = ps + qs
|
||||
offsets = tl.program_id(1).to(tl.int64) * TILE + tl.arange(0, TILE)
|
||||
token, head, channel = offsets // (H * D), offsets // D % H, offsets % D
|
||||
slot = tl.load(IDS + ps + token, token < prefix, other=0).to(tl.int64)
|
||||
kp = tl.load(KB + slot * KBS + head * KBH + channel, token < prefix, other=0)
|
||||
vp = tl.load(VB + slot * VBS + head * VBH + channel, token < prefix, other=0)
|
||||
current = qs + token - prefix
|
||||
kc = tl.load(
|
||||
K + current * KS + head * KH + channel,
|
||||
(token >= prefix) & (token < length),
|
||||
other=0,
|
||||
)
|
||||
vc = tl.load(
|
||||
V + current * VS + head * VH + channel,
|
||||
(token >= prefix) & (token < length),
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
KD + start * H * D + offsets, tl.where(token < prefix, kp, kc), token < length
|
||||
)
|
||||
tl.store(
|
||||
VD + start * H * D + offsets, tl.where(token < prefix, vp, vc), token < length
|
||||
)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(CU + row, start)
|
||||
if row == tl.num_programs(0) - 1:
|
||||
tl.store(CU + row + 1, pe + qe)
|
||||
|
||||
|
||||
def pack_prefix_current(k, v, kb, vb, qo, ki, ids, kd, vd, cu, max_length):
|
||||
h, d = k.shape[1:]
|
||||
_pack_kv[(qo.numel() - 1, triton.cdiv(max_length * h * d, 1024))](
|
||||
k,
|
||||
v,
|
||||
kb,
|
||||
vb,
|
||||
qo,
|
||||
ki,
|
||||
ids,
|
||||
kd,
|
||||
vd,
|
||||
cu,
|
||||
h,
|
||||
d,
|
||||
k.stride(0),
|
||||
k.stride(1),
|
||||
v.stride(0),
|
||||
v.stride(1),
|
||||
kb.stride(0),
|
||||
kb.stride(1),
|
||||
vb.stride(0),
|
||||
vb.stride(1),
|
||||
TILE=1024,
|
||||
)
|
||||
@@ -27,12 +27,7 @@ from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides
|
||||
from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd
|
||||
from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_gfx95_supported,
|
||||
is_gfx1250_supported,
|
||||
is_hip,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
if _is_cuda:
|
||||
@@ -1257,8 +1252,14 @@ def _fwd_kernel_unified(
|
||||
deno = tl.zeros([BLOCK_M], dtype=tl.float32)
|
||||
e_max = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
|
||||
|
||||
# Unified loop: process all KV tokens (prefix + extend)
|
||||
for start_n in range(0, cur_seq_kv_len, BLOCK_N):
|
||||
start_k, end_k = 0, cur_seq_kv_len
|
||||
if IS_CAUSAL and SLIDING_WINDOW_SIZE > 0 and not USE_CUSTOM_MASK:
|
||||
first_query = cur_block_m * BLOCK_M
|
||||
last_query = tl.minimum(first_query + BLOCK_M, cur_seq_q_len)
|
||||
start_k = tl.maximum(0, cur_seq_prefix_len + first_query - SLIDING_WINDOW_SIZE)
|
||||
start_k = start_k // BLOCK_N * BLOCK_N
|
||||
end_k = tl.minimum(cur_seq_kv_len, cur_seq_prefix_len + last_query)
|
||||
for start_n in range(start_k, end_k, BLOCK_N):
|
||||
start_n = tl.multiple_of(start_n, BLOCK_N)
|
||||
mask_n = (start_n + offs_n) < cur_seq_kv_len
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -28,6 +29,102 @@ def is_flash_attention_v4_available() -> bool:
|
||||
return _flash_attn_varlen_func is not None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_gqa_512_jit_cache():
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.interface import _get_jit_cache
|
||||
|
||||
return _get_jit_cache("fwd_gqa_512")
|
||||
|
||||
|
||||
def flash_attn_gqa_512(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
cu_seqlens_q: Optional[torch.Tensor] = None,
|
||||
cu_seqlens_k: Optional[torch.Tensor] = None,
|
||||
seqused_k: Optional[torch.Tensor] = None,
|
||||
page_table: Optional[torch.Tensor] = None,
|
||||
softmax_scale: float = 1.0,
|
||||
lse: Optional[torch.Tensor] = None,
|
||||
pack_gqa: bool = True,
|
||||
causal: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""GQA with 512-dimensional keys and separate values."""
|
||||
import cutlass.cute as cute
|
||||
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.cute_dsl_utils import (
|
||||
to_cute_tensor,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.flash_fwd_mla_sm100 import (
|
||||
FlashAttentionMLAForwardSm100,
|
||||
)
|
||||
|
||||
if cu_seqlens_q is not None:
|
||||
cu_seqlens_q = cu_seqlens_q.to(dtype=torch.int32)
|
||||
args = (
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
out,
|
||||
lse,
|
||||
softmax_scale,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_k,
|
||||
seqused_k,
|
||||
page_table,
|
||||
)
|
||||
use_cpasync = page_table is not None and k.shape[1] != 128
|
||||
key = (
|
||||
pack_gqa,
|
||||
causal,
|
||||
q.device,
|
||||
q.shape[-2] // k.shape[-2],
|
||||
k.shape[-2],
|
||||
q.shape[-1],
|
||||
v.shape[-1],
|
||||
use_cpasync,
|
||||
tuple(
|
||||
(t.ndim, t.dtype, tuple(s if s in (0, 1) else 2 for s in t.stride()))
|
||||
if isinstance(t, torch.Tensor)
|
||||
else t
|
||||
for t in args
|
||||
),
|
||||
)
|
||||
cache = _get_gqa_512_jit_cache()
|
||||
if key not in cache:
|
||||
compile_args = [
|
||||
to_cute_tensor(
|
||||
t,
|
||||
assumed_align=4
|
||||
if t.dtype in (torch.int32, torch.int64, torch.float32)
|
||||
else 16,
|
||||
)
|
||||
if isinstance(t, torch.Tensor)
|
||||
else t
|
||||
for t in args
|
||||
]
|
||||
kernel = FlashAttentionMLAForwardSm100(
|
||||
is_causal=causal,
|
||||
use_cpasync_load_KV=use_cpasync,
|
||||
is_topk_gather=False,
|
||||
pack_gqa=pack_gqa,
|
||||
qhead_per_kvhead=q.shape[-2] // k.shape[-2],
|
||||
nheads_kv=k.shape[-2],
|
||||
is_varlen_q=cu_seqlens_q is not None,
|
||||
has_qk=False,
|
||||
)
|
||||
cache[key] = cute.compile(
|
||||
kernel.forward_gqa,
|
||||
*compile_args,
|
||||
stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
cache[key](*args)
|
||||
return out
|
||||
|
||||
|
||||
def _maybe_contiguous(x: Optional[torch.Tensor]) -> Optional[torch.Tensor]:
|
||||
return x.contiguous() if x is not None and x.stride(-1) != 1 else x
|
||||
|
||||
|
||||
@@ -353,6 +353,28 @@ class FlashAttentionMLAForwardSm100:
|
||||
|
||||
return SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def forward_gqa(
|
||||
self,
|
||||
mQ: cute.Tensor,
|
||||
mK: cute.Tensor,
|
||||
mValue: cute.Tensor,
|
||||
mO: cute.Tensor,
|
||||
mLSE: Optional[cute.Tensor],
|
||||
softmax_scale: Float32,
|
||||
mCuSeqlensQ: Optional[cute.Tensor] = None,
|
||||
mCuSeqlensK: Optional[cute.Tensor] = None,
|
||||
mSeqUsedK: Optional[cute.Tensor] = None,
|
||||
mPageTable: Optional[cute.Tensor] = None,
|
||||
stream: cuda.CUstream = None,
|
||||
):
|
||||
self(
|
||||
None, mQ, None, mK, mO, mLSE, softmax_scale,
|
||||
mCuSeqlensQ=mCuSeqlensQ, mCuSeqlensK=mCuSeqlensK,
|
||||
mSeqUsedK=mSeqUsedK, mPageTable=mPageTable,
|
||||
stream=stream, mValue=mValue,
|
||||
)
|
||||
|
||||
# fmt: off
|
||||
@cute.jit
|
||||
def __call__(
|
||||
@@ -374,12 +396,14 @@ class FlashAttentionMLAForwardSm100:
|
||||
mPageTable: Optional[cute.Tensor] = None,
|
||||
window_size_left: Int32 | int | None = None,
|
||||
window_size_right: Int32 | int | None = None,
|
||||
# Always keep stream as the last parameter (EnvStream: obtained implicitly via TVM FFI).
|
||||
stream: cuda.CUstream = None,
|
||||
*,
|
||||
mValue: Optional[cute.Tensor] = None,
|
||||
):
|
||||
# fmt: on
|
||||
self.store_P = mP is not None
|
||||
self.store_row_max = mRowMax is not None
|
||||
self.separate_value = mValue is not None
|
||||
|
||||
if const_expr(self.has_qk):
|
||||
assert mQ is not None and mK is not None, "has_qk requires mQ and mK"
|
||||
@@ -402,11 +426,11 @@ class FlashAttentionMLAForwardSm100:
|
||||
*(cute.assume(s, divby=128 // mX.element_type.width) for s in mX.stride[:-1]),
|
||||
mX.stride[-1],
|
||||
)
|
||||
mQ, mQv, mK, mV, mO, mP = [
|
||||
mQ, mQv, mK, mV, mO, mP, mValue = [
|
||||
cute.make_tensor(mX.iterator, cute.make_layout(mX.shape, stride=new_stride(mX)))
|
||||
if mX is not None
|
||||
else None
|
||||
for mX in (mQ, mQv, mK, mV, mO, mP)
|
||||
for mX in (mQ, mQv, mK, mV, mO, mP, mValue)
|
||||
]
|
||||
|
||||
# (b, s, h, d) -> (s, d, h, b) or
|
||||
@@ -420,16 +444,18 @@ class FlashAttentionMLAForwardSm100:
|
||||
else None
|
||||
for mX in (mQ, mQv, mO, mP)
|
||||
]
|
||||
mK, mV = [
|
||||
mK, mV, mValue = [
|
||||
cute.make_tensor(mX.iterator, cute.select(mX.layout, mode=KV_layout_transpose))
|
||||
if mX is not None
|
||||
else None
|
||||
for mX in (mK, mV)
|
||||
for mX in (mK, mV, mValue)
|
||||
]
|
||||
# (s_k, dv, h_k, b) -> (dv, s_k, h_k, b) or
|
||||
# (total_k, dv, h_k) -> (dv, total_k, h_k)
|
||||
V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2]
|
||||
mVt = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose))
|
||||
# MLA uses the same latent for scores and values; GQA supplies separate values.
|
||||
value = mV if const_expr(mValue is None) else mValue
|
||||
mVt = cute.make_tensor(value.iterator, cute.select(value.layout, mode=V_layout_transpose))
|
||||
# (b, s_q, topk) -> (topk, s_q, b) or (total_q, topk) -> (topk, total_q)
|
||||
topk_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0]
|
||||
mIndexTopk = (
|
||||
@@ -2605,7 +2631,9 @@ class FlashAttentionMLAForwardSm100:
|
||||
|
||||
softmax = SoftmaxSm100.create(
|
||||
softmax_scale_log2,
|
||||
rescale_threshold=8.0 if const_expr(self.dtype_Q.width == 16) else 0.0,
|
||||
rescale_threshold=8.0
|
||||
if const_expr(self.dtype_Q.width == 16 and not self.separate_value)
|
||||
else 0.0,
|
||||
softmax_scale=softmax_scale,
|
||||
)
|
||||
softmax.reset()
|
||||
|
||||
+9
-1
@@ -341,9 +341,17 @@ class BlackwellFusedMultiHeadAttentionForward:
|
||||
stride_b_lse = h_r64 * h_k64 * s_lse64 if cum_seqlen_q is None else 0
|
||||
|
||||
# (s, d, ((h_r, h_k), b))
|
||||
if cutlass.const_expr(q_rank == 5):
|
||||
q_row_stride = Int64(q_tensor.stride[1])
|
||||
q_head_stride = (Int64(q_tensor.stride[3]), Int64(q_tensor.stride[2]))
|
||||
else:
|
||||
q_row_stride = Int64(q_tensor.stride[q_rank - 3])
|
||||
head_stride = Int64(q_tensor.stride[q_rank - 2])
|
||||
q_head_stride = (head_stride, head_stride * h_r64)
|
||||
q_batch_stride = Int64(q_tensor.stride[0]) if cum_seqlen_q is None else 0
|
||||
q_layout = cute.make_layout(
|
||||
(s_q_total, d, ((h_r, h_k), b)),
|
||||
stride=(d64 * h_r64 * h_k64, 1, ((d64, d64 * h_r64), stride_b_qo)),
|
||||
stride=(q_row_stride, 1, (q_head_stride, q_batch_stride)),
|
||||
)
|
||||
q = cute.make_tensor(q_tensor.iterator, q_layout)
|
||||
if cutlass.const_expr(mPageTable is not None):
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Optional, Tuple
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from triton.language.extra.cuda import libdevice
|
||||
|
||||
|
||||
def _pdl_supported() -> bool:
|
||||
@@ -60,6 +61,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
|
||||
HAS_GATE: tl.constexpr,
|
||||
MROPE: tl.constexpr,
|
||||
ENABLE_PDL: tl.constexpr,
|
||||
WEIGHT_SHIFT: tl.constexpr = 1.0,
|
||||
EXPLICIT_ROPE_FMA: tl.constexpr = False,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
head = tl.program_id(1)
|
||||
@@ -86,7 +89,7 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
|
||||
w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
|
||||
var = tl.sum(x * x, axis=0) / HEAD_DIM
|
||||
inv_rms = tl.rsqrt(var + EPS)
|
||||
x_norm = (x * inv_rms * (w + 1.0)).to(out_dtype).to(tl.float32)
|
||||
x_norm = (x * inv_rms * (w + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
|
||||
|
||||
# Pass-through tail [rotary_dim, head_dim)
|
||||
if HAS_PASS:
|
||||
@@ -104,8 +107,8 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
|
||||
wr2 = tl.load(w_ptr + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(out_dtype).to(tl.float32)
|
||||
xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(out_dtype).to(tl.float32)
|
||||
xr1 = (xr1 * inv_rms * (wr1 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
|
||||
xr2 = (xr2 * inv_rms * (wr2 + WEIGHT_SHIFT)).to(out_dtype).to(tl.float32)
|
||||
|
||||
if MROPE:
|
||||
axis = tl.load(mrope_axis_map_ptr + rot_offs, mask=rot_mask, other=0)
|
||||
@@ -121,8 +124,14 @@ def _fused_qk_rmsnorm_rope_gate_kernel(
|
||||
sin = tl.load(
|
||||
cos_sin_cache_ptr + cache_off + HALF_ROTARY + rot_offs, mask=rot_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
tl.store(out_base + rot_offs, (xr1 * cos - xr2 * sin), mask=rot_mask)
|
||||
tl.store(out_base + HALF_ROTARY + rot_offs, (xr2 * cos + xr1 * sin), mask=rot_mask)
|
||||
if EXPLICIT_ROPE_FMA:
|
||||
first = tl.fma(xr1, cos, -libdevice.mul_rn(xr2, sin))
|
||||
second = tl.fma(xr2, cos, libdevice.mul_rn(xr1, sin))
|
||||
else:
|
||||
first = xr1 * cos - xr2 * sin
|
||||
second = xr2 * cos + xr1 * sin
|
||||
tl.store(out_base + rot_offs, first, mask=rot_mask)
|
||||
tl.store(out_base + HALF_ROTARY + rot_offs, second, mask=rot_mask)
|
||||
|
||||
# Gate copy (Q heads only)
|
||||
if HAS_GATE and not is_k:
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
"""In-place Gemma QKV normalization and partial NeoX RoPE."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.fused_qk_rmsnorm_rope_gate import (
|
||||
_fused_qk_rmsnorm_rope_gate_kernel,
|
||||
)
|
||||
from sglang.kernels.ops.layernorm.gemma4_fused_ops import _gemma_qkv_rmsnorm_store
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gemma_qkv_norm_rope(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
QW,
|
||||
KW,
|
||||
CACHE,
|
||||
POS,
|
||||
SQ: tl.constexpr,
|
||||
SK: tl.constexpr,
|
||||
SV: tl.constexpr,
|
||||
SC: tl.constexpr,
|
||||
HQ: tl.constexpr,
|
||||
HK: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
R: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
FP16: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
ROT_BLOCK: tl.constexpr,
|
||||
):
|
||||
head = tl.program_id(1)
|
||||
if head < HQ + HK:
|
||||
_fused_qk_rmsnorm_rope_gate_kernel(
|
||||
Q,
|
||||
K,
|
||||
Q,
|
||||
K,
|
||||
Q,
|
||||
QW,
|
||||
KW,
|
||||
CACHE,
|
||||
POS,
|
||||
POS,
|
||||
SQ,
|
||||
SK,
|
||||
SQ,
|
||||
SK,
|
||||
SQ,
|
||||
SC,
|
||||
1,
|
||||
HQ,
|
||||
HK,
|
||||
D,
|
||||
R,
|
||||
R // 2,
|
||||
BLOCK,
|
||||
ROT_BLOCK,
|
||||
EPS,
|
||||
FP16,
|
||||
R < D,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
WEIGHT_SHIFT=0.0,
|
||||
EXPLICIT_ROPE_FMA=True,
|
||||
)
|
||||
else:
|
||||
cols = tl.arange(0, BLOCK)
|
||||
_gemma_qkv_rmsnorm_store(
|
||||
V,
|
||||
QW,
|
||||
SV,
|
||||
tl.program_id(0),
|
||||
head - HQ - HK,
|
||||
cols,
|
||||
cols < D,
|
||||
D,
|
||||
EPS,
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
def gemma_qkv_norm_rope(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cache,
|
||||
positions,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
eps,
|
||||
):
|
||||
"""Normalize Q/K with weights and V without weights, then rotate Q/K."""
|
||||
_gemma_qkv_norm_rope[(q.shape[0], num_q_heads + 2 * num_kv_heads)](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cache,
|
||||
positions,
|
||||
q.stride(0),
|
||||
k.stride(0),
|
||||
v.stride(0),
|
||||
cache.stride(0),
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
eps,
|
||||
q.dtype == torch.float16,
|
||||
triton.next_power_of_2(head_dim),
|
||||
triton.next_power_of_2(rotary_dim // 2),
|
||||
)
|
||||
@@ -33,6 +33,7 @@ def create_flashinfer_kv_indices_triton(
|
||||
req_to_token_ptr_stride,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr = 1,
|
||||
TOKEN_BLOCK_PARALLEL: tl.constexpr = False,
|
||||
token_mapping=None,
|
||||
):
|
||||
"""Gather per-request token ids into a flat CSR kv_indices stream.
|
||||
|
||||
@@ -48,6 +49,7 @@ def create_flashinfer_kv_indices_triton(
|
||||
(which bottlenecks long-context spec decode, where this kernel runs every
|
||||
iteration). With the default, the kernel is the historical
|
||||
one-program-per-request loop and 1D launch sites are unaffected.
|
||||
``token_mapping`` remaps gathered token ids in the same pass, preserving -1.
|
||||
"""
|
||||
BLOCK_SIZE: tl.constexpr = 512
|
||||
pid = tl.program_id(axis=0)
|
||||
@@ -95,6 +97,8 @@ def create_flashinfer_kv_indices_triton(
|
||||
mask=mask,
|
||||
)
|
||||
data = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
|
||||
if token_mapping is not None:
|
||||
data = tl.load(token_mapping + data, mask=mask & (data >= 0), other=-1)
|
||||
tl.store(kv_indices_ptr + kv_indices_offset + offset, data, mask=mask)
|
||||
|
||||
|
||||
|
||||
@@ -580,3 +580,18 @@ register_kernel(
|
||||
target="sglang.kernels.ops.layernorm.rms_normalize_hip:rms_normalize_triton",
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="layernorm.rmsnorm_fanout",
|
||||
backend=KernelBackend.CUTE_DSL,
|
||||
target="sglang.kernels.ops.layernorm.rmsnorm_fanout:rmsnorm_fanout",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
|
||||
),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=_NORM_DTYPES,
|
||||
description="Three weighted RMSNorm outputs sharing a single-CTA row reduction.",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -10,6 +10,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import is_sm100_supported
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gemma_rmsnorm_residual_kernel(
|
||||
@@ -277,7 +279,7 @@ def gemma_qkv_rmsnorm(
|
||||
assert k.stride(-1) == 1 and v.stride(-1) == 1
|
||||
assert k_weight is not None and k_weight.shape[-1] == head_dim
|
||||
|
||||
if M <= 256:
|
||||
if M <= 256 or (M == 512 and is_sm100_supported()):
|
||||
total_heads = num_q_heads + (2 * num_kv_heads if has_kv else 0)
|
||||
_gemma_qkv_rmsnorm_kernel[(M, total_heads)](
|
||||
q,
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Copyright 2025 FlashInfer team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
"""Three RMSNorm outputs sharing FlashInfer's input load and reduction.
|
||||
|
||||
Uses the CuTe RMSNormKernel layout/reduction interface from FlashInfer 0.6.18.
|
||||
The row-count dispatch covers the measured decode buckets on Blackwell.
|
||||
"""
|
||||
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
|
||||
_FANOUT_MAX_ROWS = 2048
|
||||
|
||||
|
||||
def can_use_rmsnorm_fanout(x):
|
||||
return (
|
||||
0 < x.shape[0] <= _FANOUT_MAX_ROWS
|
||||
and _fanout_kernel(x.dtype, x.shape[1]) is not None
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _fanout_kernel(dtype, width):
|
||||
if dtype not in (torch.bfloat16, torch.float16):
|
||||
return None
|
||||
kernel = _fanout_kernel_type()(_cutlass_dtype(dtype), width)
|
||||
return kernel if kernel.cluster_n == 1 else None
|
||||
|
||||
|
||||
def _cutlass_dtype(dtype):
|
||||
import cutlass
|
||||
|
||||
return cutlass.BFloat16 if dtype == torch.bfloat16 else cutlass.Float16
|
||||
|
||||
|
||||
def rmsnorm_fanout(x, weight0, weight1, weight2, eps):
|
||||
"""Normalize contiguous FP16/BF16 rows with three matching [H] weights."""
|
||||
outputs = tuple(torch.empty_like(x) for _ in range(3))
|
||||
compiled = _compile_fanout(x.device, x.dtype, x.shape[1])
|
||||
compiled(x, weight0, weight1, weight2, *outputs, x.shape[0], eps)
|
||||
return outputs
|
||||
|
||||
|
||||
@cache
|
||||
def _compile_fanout(device, torch_dtype, width):
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
|
||||
dtype = _cutlass_dtype(torch_dtype)
|
||||
kernel = _fanout_kernel(torch_dtype, width)
|
||||
rows = cute.sym_int(64)
|
||||
matrix = cute.runtime.make_fake_compact_tensor(
|
||||
dtype, (rows, width), stride_order=(1, 0), assumed_align=16
|
||||
)
|
||||
weight = cute.runtime.make_fake_compact_tensor(dtype, (width,), assumed_align=16)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
matrix,
|
||||
weight,
|
||||
weight,
|
||||
weight,
|
||||
matrix,
|
||||
matrix,
|
||||
matrix,
|
||||
cutlass.Int64(1),
|
||||
cutlass.Float32(1e-6),
|
||||
stream=cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _fanout_kernel_type():
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cutlass import Float32, Int64
|
||||
from flashinfer.norm.kernels.rmsnorm import RMSNormKernel
|
||||
from flashinfer.norm.utils import predicate_k, row_reduce_sum_multirow
|
||||
|
||||
class RMSNormFanout(RMSNormKernel):
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
x: cute.Tensor,
|
||||
w0: cute.Tensor,
|
||||
w1: cute.Tensor,
|
||||
w2: cute.Tensor,
|
||||
y0: cute.Tensor,
|
||||
y1: cute.Tensor,
|
||||
y2: cute.Tensor,
|
||||
m: Int64,
|
||||
eps: Float32,
|
||||
stream,
|
||||
):
|
||||
shape, stride = self._make_tv_layout(
|
||||
self.threads_per_row,
|
||||
self.rows_per_block,
|
||||
self.vec_size,
|
||||
self.num_vec_blocks,
|
||||
)
|
||||
layout = cute.make_layout(shape, stride=stride)
|
||||
tile = (self.rows_per_block, self.cols_per_tile)
|
||||
self.kernel(x, w0, w1, w2, y0, y1, y2, m, eps, layout, tile).launch(
|
||||
grid=[cute.ceil_div(m, self.rows_per_block), 1, 1],
|
||||
block=[self.num_threads, 1, 1],
|
||||
smem=self._smem_size_in_bytes(),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
x: cute.Tensor,
|
||||
w0: cute.Tensor,
|
||||
w1: cute.Tensor,
|
||||
w2: cute.Tensor,
|
||||
y0: cute.Tensor,
|
||||
y1: cute.Tensor,
|
||||
y2: cute.Tensor,
|
||||
m: Int64,
|
||||
eps: Float32,
|
||||
layout: cute.Layout,
|
||||
tile: cute.Shape,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
block, _, _ = cute.arch.block_idx()
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
if cutlass.const_expr(self.use_async_copy):
|
||||
sx = smem.allocate_tensor(
|
||||
x.element_type,
|
||||
cute.make_ordered_layout(tile, order=(1, 0)),
|
||||
byte_alignment=16,
|
||||
)
|
||||
reduction = smem.allocate_tensor(
|
||||
Float32,
|
||||
cute.make_layout((self.rows_per_block, self.warps_per_row)),
|
||||
byte_alignment=4,
|
||||
)
|
||||
gx = cute.local_tile(x, tile, (block, 0))
|
||||
coords = cute.local_tile(
|
||||
cute.make_identity_tensor(x.shape), tile, (block, 0)
|
||||
)
|
||||
sync = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(),
|
||||
x.element_type,
|
||||
num_bits_per_copy=self.copy_bits,
|
||||
)
|
||||
if cutlass.const_expr(self.use_async_copy):
|
||||
async_copy = cute.make_copy_atom(
|
||||
cute.nvgpu.cpasync.CopyG2SOp(),
|
||||
x.element_type,
|
||||
num_bits_per_copy=self.copy_bits,
|
||||
)
|
||||
loader = cute.make_tiled_copy(async_copy, layout, tile)
|
||||
else:
|
||||
loader = cute.make_tiled_copy(sync, layout, tile)
|
||||
copier = cute.make_tiled_copy(sync, layout, tile)
|
||||
thread_x = loader.get_slice(tid)
|
||||
thread_o = copier.get_slice(tid)
|
||||
txg = thread_x.partition_S(gx)
|
||||
txc = thread_x.partition_S(coords)
|
||||
txr = cute.make_fragment_like(txg)
|
||||
x_pred = predicate_k(txc, limit=self.H)
|
||||
row_valid = txc[(0, 0), 0, 0][0] < m
|
||||
if cutlass.const_expr(self.use_async_copy):
|
||||
txs = thread_x.partition_D(sx)
|
||||
if row_valid:
|
||||
cute.copy(async_copy, txg, txs, pred=x_pred)
|
||||
cute.arch.cp_async_commit_group()
|
||||
cute.arch.cp_async_wait_group(0)
|
||||
cute.autovec_copy(txs, txr)
|
||||
else:
|
||||
txr.store(cute.zeros_like(txr, dtype=x.element_type))
|
||||
if row_valid:
|
||||
cute.copy(sync, txg, txr, pred=x_pred)
|
||||
value = txr.load().to(Float32)
|
||||
summed = row_reduce_sum_multirow(
|
||||
value * value, self.threads_per_row, reduction, None, 1
|
||||
)
|
||||
rstd = cute.math.rsqrt(summed / Float32(self.H) + eps, fastmath=True)
|
||||
cute.arch.barrier()
|
||||
if cutlass.const_expr(self.use_async_copy):
|
||||
cute.autovec_copy(txs, txr)
|
||||
value = txr.load().to(Float32)
|
||||
weights = (w0, w1, w2)
|
||||
outputs = (y0, y1, y2)
|
||||
wpred = predicate_k(thread_o.partition_S(coords), limit=self.H)
|
||||
for index in cutlass.range_constexpr(3):
|
||||
weight = weights[index]
|
||||
expanded = cute.make_tensor(
|
||||
weight.iterator,
|
||||
cute.prepend(
|
||||
weight.layout, cute.make_layout((tile[0],), stride=(0,))
|
||||
),
|
||||
)
|
||||
gw = cute.local_tile(expanded, tile, (0, 0))
|
||||
twg = thread_o.partition_S(gw)
|
||||
twr = cute.make_fragment_like(twg)
|
||||
cute.copy(sync, twg, twr, pred=wpred)
|
||||
weight_value = thread_x.retile(twr).load().to(Float32)
|
||||
gy = cute.local_tile(outputs[index], tile, (block, 0))
|
||||
tog = thread_o.partition_D(gy)
|
||||
tor = cute.make_fragment_like(tog)
|
||||
result = value * rstd * (weight_value + Float32(0.0))
|
||||
tor.store(result.to(outputs[index].element_type))
|
||||
if row_valid:
|
||||
cute.copy(sync, tor, tog, pred=x_pred)
|
||||
|
||||
return RMSNormFanout
|
||||
@@ -111,3 +111,17 @@ register_kernel(
|
||||
target="sglang.kernels.ops.sampling.murmur_hash:murmur_hash32",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="sampling.denoiser_statistics",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.sampling.denoiser_statistics:denoiser_statistics",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float32",),
|
||||
description="FP32 probabilities, entropy and raw-logit argmax for contiguous [B,M,V] logits; optional BF16 probabilities.",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Chunked FP32 statistics for large-vocabulary block denoising.
|
||||
|
||||
The partials carry a maximum, exponential sum and centered first moment.
|
||||
Merging these computes entropy without materializing log-probabilities. A
|
||||
separate pass writes FP32 probabilities for the existing categorical sampler
|
||||
and self-conditioning GEMM. Sampling and its generator state stay unchanged.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _denoiser_partial_statistics(
|
||||
LOGITS,
|
||||
TEMPERATURES,
|
||||
PARTIALS,
|
||||
PARTIAL_ARGMAX,
|
||||
VOCAB: tl.constexpr,
|
||||
CANVAS: tl.constexpr,
|
||||
ROWS: tl.constexpr,
|
||||
CHUNKS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
part = tl.program_id(1)
|
||||
j = part * BLOCK + tl.arange(0, BLOCK)
|
||||
temp = tl.load(TEMPERATURES + row // CANVAS)
|
||||
raw = tl.load(
|
||||
LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf")
|
||||
).to(tl.float32)
|
||||
# Approximate reciprocal scaling changes sharp distributions enough to
|
||||
# exceed the statistics tolerance. Match FP32 division rounding.
|
||||
scaled = tl.div_rn(raw, temp)
|
||||
mx = tl.max(scaled, 0)
|
||||
d = scaled - tl.where(mx == -float("inf"), 0.0, mx)
|
||||
e = tl.exp(d)
|
||||
z = tl.sum(e, 0)
|
||||
s = tl.sum(tl.where(j < VOCAB, d * e, 0.0), 0)
|
||||
# Preserve PyTorch argmax's first-NaN and first-maximum behavior.
|
||||
# An explicit NaN reduction is required; testing the reduced maximum
|
||||
# itself is not equivalent after Triton lowering.
|
||||
rawmax = tl.max(raw, 0)
|
||||
has_nan = tl.sum(((j < VOCAB) & (raw != raw)).to(tl.int32), 0) > 0
|
||||
idx = tl.min(
|
||||
tl.where(
|
||||
(j < VOCAB) & tl.where(has_nan, raw != raw, raw == rawmax), j, 2147483647
|
||||
),
|
||||
0,
|
||||
)
|
||||
off = row * CHUNKS + part
|
||||
tl.store(PARTIALS + off, mx)
|
||||
tl.store(PARTIALS + ROWS * CHUNKS + off, z)
|
||||
tl.store(PARTIALS + 2 * ROWS * CHUNKS + off, s)
|
||||
tl.store(PARTIALS + 3 * ROWS * CHUNKS + off, rawmax)
|
||||
tl.store(PARTIAL_ARGMAX + off, idx)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _denoiser_merge_statistics(
|
||||
PARTIALS,
|
||||
PARTIAL_ARGMAX,
|
||||
NORMALIZER,
|
||||
ENTROPY,
|
||||
ARGMAX,
|
||||
ROWS: tl.constexpr,
|
||||
CHUNKS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
p = tl.arange(0, BLOCK)
|
||||
off = row * CHUNKS + p
|
||||
mx = tl.load(PARTIALS + off, mask=p < CHUNKS, other=-float("inf"))
|
||||
z = tl.load(PARTIALS + ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0)
|
||||
moment = tl.load(PARTIALS + 2 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=0.0)
|
||||
maximum = tl.max(mx, 0)
|
||||
shift = mx - maximum
|
||||
alpha = tl.exp(shift)
|
||||
denom = tl.sum(alpha * z, 0)
|
||||
total = tl.sum(
|
||||
tl.where(p < CHUNKS, alpha * (moment + tl.where(z > 0, z * shift, 0.0)), 0.0), 0
|
||||
)
|
||||
logz = tl.log(denom)
|
||||
tl.store(NORMALIZER + row, maximum)
|
||||
tl.store(NORMALIZER + ROWS + row, logz)
|
||||
tl.store(ENTROPY + row, logz - total / denom)
|
||||
rawmax = tl.load(
|
||||
PARTIALS + 3 * ROWS * CHUNKS + off, mask=p < CHUNKS, other=-float("inf")
|
||||
)
|
||||
best = tl.max(rawmax, 0)
|
||||
idx = tl.load(PARTIAL_ARGMAX + off, mask=p < CHUNKS, other=2147483647)
|
||||
has_nan = tl.sum(((p < CHUNKS) & (rawmax != rawmax)).to(tl.int32), 0) > 0
|
||||
tl.store(
|
||||
ARGMAX + row,
|
||||
tl.min(
|
||||
tl.where(
|
||||
(p < CHUNKS) & tl.where(has_nan, rawmax != rawmax, rawmax == best),
|
||||
idx,
|
||||
2147483647,
|
||||
),
|
||||
0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _denoiser_write_probabilities(
|
||||
LOGITS,
|
||||
TEMPERATURES,
|
||||
NORMALIZER,
|
||||
PROBABILITIES,
|
||||
SOFT_PROBABILITIES,
|
||||
VOCAB: tl.constexpr,
|
||||
CANVAS: tl.constexpr,
|
||||
ROWS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
WRITE_SOFT: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
j = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
temp = tl.load(TEMPERATURES + row // CANVAS)
|
||||
mx = tl.load(NORMALIZER + row)
|
||||
logz = tl.load(NORMALIZER + ROWS + row)
|
||||
raw = tl.load(
|
||||
LOGITS + row.to(tl.int64) * VOCAB + j, mask=j < VOCAB, other=-float("inf")
|
||||
).to(tl.float32)
|
||||
p = tl.exp(tl.div_rn(raw, temp) - mx - logz)
|
||||
tl.store(PROBABILITIES + row.to(tl.int64) * VOCAB + j, p, mask=j < VOCAB)
|
||||
if WRITE_SOFT:
|
||||
tl.store(
|
||||
SOFT_PROBABILITIES + row.to(tl.int64) * VOCAB + j,
|
||||
p.to(tl.bfloat16),
|
||||
mask=j < VOCAB,
|
||||
)
|
||||
|
||||
|
||||
def denoiser_statistics(
|
||||
logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
soft_probabilities: torch.Tensor | None = None,
|
||||
):
|
||||
"""Statistics for contiguous FP32 [B, M, V] logits and [B] temperatures.
|
||||
|
||||
soft_probabilities, when supplied, is a contiguous BF16 buffer shaped like logits.
|
||||
"""
|
||||
assert logits.ndim == 3 and temperatures.shape == (logits.shape[0],)
|
||||
block = 4096
|
||||
batch, canvas, vocab = logits.shape
|
||||
rows = batch * canvas
|
||||
parts = triton.cdiv(vocab, block)
|
||||
partials = torch.empty((4, rows, parts), device=logits.device, dtype=torch.float32)
|
||||
partial_argmax = torch.empty((rows, parts), device=logits.device, dtype=torch.int32)
|
||||
normalizer = torch.empty((2, rows), device=logits.device, dtype=torch.float32)
|
||||
probabilities = torch.empty_like(logits, dtype=torch.float32)
|
||||
entropies = torch.empty((batch, canvas), device=logits.device, dtype=torch.float32)
|
||||
argmax = torch.empty((batch, canvas), device=logits.device, dtype=torch.int64)
|
||||
_denoiser_partial_statistics[(rows, parts)](
|
||||
logits,
|
||||
temperatures,
|
||||
partials,
|
||||
partial_argmax,
|
||||
vocab,
|
||||
canvas,
|
||||
rows,
|
||||
parts,
|
||||
block,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
_denoiser_merge_statistics[(rows,)](
|
||||
partials,
|
||||
partial_argmax,
|
||||
normalizer,
|
||||
entropies,
|
||||
argmax,
|
||||
rows,
|
||||
parts,
|
||||
triton.next_power_of_2(parts),
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
_denoiser_write_probabilities[(rows, triton.cdiv(vocab, block))](
|
||||
logits,
|
||||
temperatures,
|
||||
normalizer,
|
||||
probabilities,
|
||||
soft_probabilities if soft_probabilities is not None else probabilities,
|
||||
vocab,
|
||||
canvas,
|
||||
rows,
|
||||
block,
|
||||
soft_probabilities is not None,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return probabilities, entropies, argmax
|
||||
@@ -6,7 +6,12 @@ The Triton kernels migrated here live in this package
|
||||
"""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import KernelBackend, KernelSpec
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
|
||||
# (module, public_fn) migrated from speculative/triton_ops.
|
||||
_TRITON_KERNELS = [
|
||||
@@ -37,4 +42,17 @@ for _mod, _fn in _TRITON_KERNELS:
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="speculative.div_argmax",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.speculative.row_argmax:div_argmax",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float32",),
|
||||
description="Row-wise division and argmax on matching 2D tensors with unit column strides.",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
__all__ = []
|
||||
|
||||
@@ -68,11 +68,16 @@ def _medium_argmax_partial_kernel(
|
||||
SX: tl.constexpr,
|
||||
SPLITS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
DIVISOR=None,
|
||||
SD: tl.constexpr = 0,
|
||||
):
|
||||
row, part = tl.program_id(0), tl.program_id(1)
|
||||
ix = part * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = ix < N
|
||||
v = tl.load(X + row * SX + ix, valid, float("-inf"))
|
||||
if DIVISOR is not None:
|
||||
divisor = tl.load(DIVISOR + row * SD + ix, valid, 1.0)
|
||||
v = tl.div_rn(v, divisor)
|
||||
i = tl.where(valid, ix, N)
|
||||
best_v, best_i = tl.reduce((v, i), 0, _argmax_pair)
|
||||
tl.store(PV + row * SPLITS + part, best_v)
|
||||
@@ -89,25 +94,50 @@ def _medium_argmax_final_kernel(PV, PI, OUT, SPLITS: tl.constexpr, BLOCK: tl.con
|
||||
tl.store(OUT + row, index.to(tl.int64))
|
||||
|
||||
|
||||
def div_argmax(x: torch.Tensor, divisor: torch.Tensor) -> torch.Tensor:
|
||||
"""FP32 row-wise ``(x / divisor).argmax(-1)`` with unit column strides."""
|
||||
assert x.ndim == 2 and x.dtype == torch.float32 and x.stride(1) == 1
|
||||
assert (
|
||||
divisor.shape == x.shape
|
||||
and divisor.dtype == x.dtype
|
||||
and divisor.device == x.device
|
||||
and divisor.stride(1) == 1
|
||||
)
|
||||
return _medium_row_argmax(x, divisor)
|
||||
|
||||
|
||||
def _medium_row_argmax(x, divisor=None):
|
||||
rows, n = x.shape
|
||||
block = 4096 if rows <= 256 else 8192
|
||||
splits = triton.cdiv(n, block)
|
||||
values = torch.empty((rows, splits), dtype=torch.float32, device=x.device)
|
||||
indices = torch.empty((rows, splits), dtype=torch.int32, device=x.device)
|
||||
out = torch.empty(rows, dtype=torch.int64, device=x.device)
|
||||
_medium_argmax_partial_kernel[(rows, splits)](
|
||||
x,
|
||||
values,
|
||||
indices,
|
||||
n,
|
||||
x.stride(0),
|
||||
splits,
|
||||
block,
|
||||
DIVISOR=divisor,
|
||||
SD=divisor.stride(0) if divisor is not None else 0,
|
||||
num_warps=4,
|
||||
)
|
||||
_medium_argmax_final_kernel[(rows,)](
|
||||
values, indices, out, splits, triton.next_power_of_2(splits), num_warps=1
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def row_argmax(x: torch.Tensor) -> torch.Tensor:
|
||||
"""``x.argmax(dim=-1)`` for FP32 speculative logits with a wide vocab."""
|
||||
assert x.dim() == 2 and x.dtype == torch.float32 and x.stride(1) == 1
|
||||
rows, n = x.shape
|
||||
out = torch.empty((rows,), dtype=torch.int64, device=x.device)
|
||||
if rows > 64:
|
||||
# Whole aligned tiles avoid the loop and unaligned partition starts.
|
||||
# Larger batches use wider tiles to limit the total number of CTAs.
|
||||
block = 4096 if rows <= 256 else 8192
|
||||
splits = triton.cdiv(n, block)
|
||||
pv = torch.empty((rows, splits), dtype=torch.float32, device=x.device)
|
||||
pi = torch.empty((rows, splits), dtype=torch.int32, device=x.device)
|
||||
_medium_argmax_partial_kernel[(rows, splits)](
|
||||
x, pv, pi, n, x.stride(0), splits, block, num_warps=4
|
||||
)
|
||||
_medium_argmax_final_kernel[(rows,)](
|
||||
pv, pi, out, splits, triton.next_power_of_2(splits), num_warps=1
|
||||
)
|
||||
return out
|
||||
return _medium_row_argmax(x)
|
||||
out = torch.empty((rows,), dtype=torch.int64, device=x.device)
|
||||
pv = torch.empty((rows, _SPLITS), dtype=torch.float32, device=x.device)
|
||||
pi = torch.empty((rows, _SPLITS), dtype=torch.int32, device=x.device)
|
||||
_argmax_partial_kernel[(rows, _SPLITS)](
|
||||
|
||||
@@ -26,11 +26,10 @@ def handle_dllm_inference(server_args: Any):
|
||||
return
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
run_post_process_pass(server_args, _dllm_attention_backend)
|
||||
get_algorithm_cls(cfg.dllm_algorithm).configure_server_args(server_args)
|
||||
|
||||
# On AMD/HIP, disable cuda graph for DLLM (the attention_backend
|
||||
# resolution moved to the pipeline: arg_groups/overrides.py
|
||||
# _dllm_attention_backend, invoked below at its legacy slot).
|
||||
# DLLM CUDA graphs are disabled on AMD/HIP.
|
||||
if get_platform().is_hip:
|
||||
if (
|
||||
cfg.cuda_graph_config.decode.backend != Backend.DISABLED
|
||||
@@ -54,7 +53,6 @@ def handle_dllm_inference(server_args: Any):
|
||||
),
|
||||
)
|
||||
|
||||
run_post_process_pass(server_args, _dllm_attention_backend)
|
||||
run_post_process_pass(server_args, _dllm_overlap_disable)
|
||||
|
||||
# The page-size alignment + block-size cap for dllm moved to the
|
||||
|
||||
@@ -1231,6 +1231,11 @@ def _fa4_page_constraint(view: Any) -> dict:
|
||||
# page_size==1, so skip the 128 auto-force for it and keep the default.
|
||||
and (view.speculative_eagle_topk or 0) <= 1
|
||||
):
|
||||
if (
|
||||
"DiffusionGemmaForBlockDiffusion"
|
||||
in model_config_of(view).hf_config.architectures
|
||||
):
|
||||
return {"page_size": 1}
|
||||
logger.warning(
|
||||
f"FA4 backend only supports page size 128 for non-MLA model architectures, changing page_size from {view.page_size} to 128."
|
||||
)
|
||||
@@ -1586,6 +1591,12 @@ def _dllm_attention_backend(view: Any) -> dict:
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
algorithm_cls = get_algorithm_cls(view.dllm_algorithm)
|
||||
if view.attention_backend in algorithm_cls.supported_attention_backends and all(
|
||||
getattr(view, field, None)
|
||||
in (None, *algorithm_cls.supported_attention_backends)
|
||||
for field in ("prefill_attention_backend", "decode_attention_backend")
|
||||
):
|
||||
return {}
|
||||
if backend := algorithm_cls.required_attention_backend:
|
||||
fields = (
|
||||
"attention_backend",
|
||||
|
||||
@@ -34,6 +34,9 @@ class DllmAlgorithm:
|
||||
supported_architectures: Tuple[str, ...] = ()
|
||||
requires_separate_context_encoding = False
|
||||
required_attention_backend: Optional[str] = None
|
||||
reuse_forward_metadata: bool = False
|
||||
supported_attention_backends: Tuple[str, ...] = ()
|
||||
capture_input_preparation: bool = False
|
||||
|
||||
def __init__(self, config: DllmConfig):
|
||||
self.block_size = config.block_size
|
||||
@@ -113,11 +116,9 @@ class DllmAlgorithm:
|
||||
if all(start == self.block_size for start in start_list):
|
||||
return out.logits_output, [], None, None, out.can_run_graph
|
||||
|
||||
# NPU: attention metadata is stable across a block's denoise steps (the
|
||||
# first forward above already planned it), so mark it ready once and let
|
||||
# every later forward skip re-planning.
|
||||
if _is_npu:
|
||||
forward_batch.mark_forward_metadata_ready()
|
||||
# The first forward has planned the fixed canvas's attention metadata.
|
||||
if _is_npu or (self.reuse_forward_metadata and out.can_run_graph):
|
||||
forward_batch.mark_forward_metadata_ready(replan_equivalent=not _is_npu)
|
||||
for _ in range(self.max_steps(self.block_size)):
|
||||
done = self.step(forward_batch, out.logits_output.full_logits, states)
|
||||
if all(done):
|
||||
|
||||
@@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import model_config_of
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
attention_backends_of,
|
||||
model_config_of,
|
||||
)
|
||||
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.dllm.algorithm.base import DllmAlgorithm, DllmRunOutput
|
||||
@@ -31,11 +34,39 @@ def _denoiser_statistics(logits: torch.Tensor, temperatures: torch.Tensor):
|
||||
_compiled_denoiser_statistics = torch.compile(_denoiser_statistics, dynamic=True)
|
||||
|
||||
|
||||
def _use_split_denoiser_statistics(logits, temperatures):
|
||||
return (
|
||||
logits.dtype == temperatures.dtype == torch.float32
|
||||
and logits.is_contiguous()
|
||||
and temperatures.is_contiguous()
|
||||
and logits.shape[-1] >= 65536
|
||||
and logits.shape[0] * logits.shape[1] >= 32
|
||||
and torch.cuda.get_device_capability(logits.device)[0] == 10
|
||||
)
|
||||
|
||||
|
||||
def _denoiser_statistics_cuda(logits, temperatures, *, write_soft=False):
|
||||
soft_probabilities = None
|
||||
if _use_split_denoiser_statistics(logits, temperatures):
|
||||
from sglang.kernels.ops.sampling.denoiser_statistics import denoiser_statistics
|
||||
|
||||
if write_soft:
|
||||
soft_probabilities = torch.empty_like(logits, dtype=torch.bfloat16)
|
||||
result = denoiser_statistics(logits, temperatures, soft_probabilities)
|
||||
else:
|
||||
result = _compiled_denoiser_statistics(logits, temperatures)
|
||||
return *result, soft_probabilities
|
||||
|
||||
|
||||
def _sample_denoiser(probabilities: torch.Tensor, generator: torch.Generator):
|
||||
# The exponential-race formulation samples the same categorical distribution
|
||||
# as multinomial. Probabilities come from softmax, so multinomial's repeated
|
||||
# validation and device-to-host synchronization are unnecessary here.
|
||||
noise = torch.empty_like(probabilities).exponential_(1.0, generator=generator)
|
||||
if probabilities.is_cuda:
|
||||
from sglang.kernels.ops.speculative.row_argmax import div_argmax
|
||||
|
||||
return div_argmax(probabilities, noise)
|
||||
return (probabilities / noise).argmax(dim=-1)
|
||||
|
||||
|
||||
@@ -43,6 +74,9 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
supported_architectures = ("DiffusionGemmaForBlockDiffusion",)
|
||||
requires_separate_context_encoding = True
|
||||
required_attention_backend = "triton"
|
||||
reuse_forward_metadata = True
|
||||
supported_attention_backends = ("triton", "fa4")
|
||||
capture_input_preparation = True
|
||||
|
||||
@classmethod
|
||||
def configure_server_args(cls, server_args: ServerArgs) -> None:
|
||||
@@ -66,16 +100,24 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
):
|
||||
raise ValueError("DiffusionGemma requires tied word embeddings")
|
||||
|
||||
graph_config = cfg.cuda_graph_config
|
||||
if graph_config.prefill.backend == Backend.FULL:
|
||||
prefill_backend, _ = attention_backends_of(cfg)
|
||||
if prefill_backend != "fa4":
|
||||
raise ValueError(
|
||||
"Gemma4Renoise FULL prefill CUDA graphs require prefill "
|
||||
f"backend 'fa4'; got '{prefill_backend}'."
|
||||
)
|
||||
else:
|
||||
graph_config = with_phase(
|
||||
graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
cls.__name__,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config,
|
||||
Phase.PREFILL,
|
||||
backend=Backend.DISABLED,
|
||||
),
|
||||
cuda_graph_config=graph_config,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -153,6 +195,11 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
self.seed = algorithm_config.get("seed")
|
||||
self.vocab_size = None
|
||||
self.embed_tokens = None
|
||||
self._temperature_tables = {}
|
||||
self._temperature_buffers = {}
|
||||
self._zero_signals = {}
|
||||
self.use_temperature_table = algorithm_config.get("use_temperature_table", True)
|
||||
self.use_graph_input_preparation = config.capture_input_preparation
|
||||
|
||||
if (
|
||||
not isinstance(self.max_denoising_steps, int)
|
||||
@@ -226,40 +273,86 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
"argmax": current,
|
||||
"history": [],
|
||||
"self_conditioning": None,
|
||||
"rng_state": generator.get_state(),
|
||||
"generator": generator,
|
||||
"finished": False,
|
||||
}
|
||||
)
|
||||
return states
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_inputs(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
forward_batch: ForwardBatch,
|
||||
states: List[Any],
|
||||
) -> None:
|
||||
self._write_input_ids(forward_batch, states)
|
||||
if self.fdfo or all(
|
||||
state["step"] == self.max_denoising_steps for state in states
|
||||
):
|
||||
self._write_input_ids(forward_batch, states)
|
||||
|
||||
signals = [state["self_conditioning"] for state in states]
|
||||
signal = next((value for value in signals if value is not None), None)
|
||||
if signal is not None:
|
||||
batched = torch.zeros(
|
||||
(len(states), self.block_size, signal.shape[-1]),
|
||||
dtype=signal.dtype,
|
||||
device=signal.device,
|
||||
if signal is not None or self.use_graph_input_preparation:
|
||||
weight = self.embed_tokens.weight
|
||||
key = (weight.device, weight.dtype, weight.shape[-1])
|
||||
if key not in self._zero_signals:
|
||||
self._zero_signals[key] = weight.new_zeros(
|
||||
self.block_size, weight.shape[-1]
|
||||
)
|
||||
zero = self._zero_signals[key]
|
||||
shape = (len(states) * self.block_size, weight.shape[-1])
|
||||
batched = forward_batch.dllm_input_preparation_state
|
||||
if batched is None or batched.shape != shape:
|
||||
batched = weight.new_empty(shape)
|
||||
torch.stack(
|
||||
[zero if value is None else value for value in signals],
|
||||
out=batched.view(len(states), self.block_size, -1),
|
||||
)
|
||||
for index, value in enumerate(signals):
|
||||
if value is not None:
|
||||
batched[index].copy_(value)
|
||||
signal = batched.view(len(states) * self.block_size, -1)
|
||||
forward_batch.dllm_input_preparation_state = batched
|
||||
signal = batched
|
||||
|
||||
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
|
||||
forward_batch.input_ids, signal
|
||||
if self.use_graph_input_preparation:
|
||||
forward_batch.input_embeds = None
|
||||
else:
|
||||
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
|
||||
forward_batch.input_ids, signal
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def prepare_graph_inputs(model, input_ids, state):
|
||||
return model.prepare_dllm_input_embeds(input_ids, state)
|
||||
|
||||
def _temperatures(self, logits, states):
|
||||
if not self.use_temperature_table:
|
||||
return logits.new_tensor(
|
||||
[self._temperature(state["step"]) for state in states]
|
||||
)
|
||||
key = (logits.device, logits.dtype)
|
||||
if key not in self._temperature_tables:
|
||||
self._temperature_tables[key] = logits.new_tensor(
|
||||
[
|
||||
self._temperature(step)
|
||||
for step in range(self.max_denoising_steps + 1)
|
||||
]
|
||||
)
|
||||
table = self._temperature_tables[key]
|
||||
if len(states) == 1:
|
||||
step = states[0]["step"]
|
||||
return table[step : step + 1]
|
||||
buffer_key = (*key, len(states))
|
||||
if buffer_key not in self._temperature_buffers:
|
||||
self._temperature_buffers[buffer_key] = logits.new_empty(len(states))
|
||||
return torch.stack(
|
||||
[table[state["step"]] for state in states],
|
||||
out=self._temperature_buffers[buffer_key],
|
||||
)
|
||||
|
||||
def _write_input_ids(self, forward_batch: ForwardBatch, states: List[Any]) -> None:
|
||||
current = torch.stack([state["current"] for state in states])
|
||||
forward_batch.input_ids.copy_(current.view(-1))
|
||||
torch.stack(
|
||||
[state["current"] for state in states],
|
||||
out=forward_batch.input_ids.view(len(states), self.block_size),
|
||||
)
|
||||
|
||||
def _soft_embeddings(self, probabilities: torch.Tensor) -> torch.Tensor:
|
||||
weight = self.embed_tokens.weight
|
||||
@@ -292,13 +385,23 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
self._write_input_ids(forward_batch, states)
|
||||
return [True] * len(states)
|
||||
|
||||
temperatures = logits.new_tensor(
|
||||
[self._temperature(state["step"]) for state in states]
|
||||
)
|
||||
statistics = (
|
||||
_compiled_denoiser_statistics if logits.is_cuda else _denoiser_statistics
|
||||
)
|
||||
probabilities, token_entropies, argmax_tokens = statistics(logits, temperatures)
|
||||
temperatures = self._temperatures(logits, states)
|
||||
soft_probabilities = None
|
||||
if logits.is_cuda:
|
||||
probabilities, token_entropies, argmax_tokens, soft_probabilities = (
|
||||
_denoiser_statistics_cuda(
|
||||
logits,
|
||||
temperatures,
|
||||
write_soft=(
|
||||
self.embed_tokens.weight.dtype == torch.bfloat16
|
||||
and getattr(self.embed_tokens, "tp_size", 1) == 1
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
probabilities, token_entropies, argmax_tokens = _denoiser_statistics(
|
||||
logits, temperatures
|
||||
)
|
||||
sorted_entropy, indices = torch.sort(token_entropies, dim=-1)
|
||||
cumulative_entropy = torch.cumsum(sorted_entropy, dim=-1)
|
||||
sorted_selected = cumulative_entropy - sorted_entropy <= self.entropy_bound
|
||||
@@ -312,8 +415,7 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
if state["finished"]:
|
||||
continue
|
||||
active.append(index)
|
||||
generator = torch.Generator(device=logits.device)
|
||||
generator.set_state(state["rng_state"])
|
||||
generator = state["generator"]
|
||||
denoiser = _sample_denoiser(probabilities[index], generator)
|
||||
random_canvas = torch.randint(
|
||||
self.vocab_size,
|
||||
@@ -327,7 +429,8 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
if self.stability_threshold == 0:
|
||||
stable = torch.ones((), dtype=torch.bool, device=logits.device)
|
||||
elif len(history) == self.stability_threshold:
|
||||
stable = (torch.stack(history) == argmax).all()
|
||||
previous = history[0] if len(history) == 1 else torch.stack(history)
|
||||
stable = (previous == argmax).all()
|
||||
else:
|
||||
stable = torch.zeros((), dtype=torch.bool, device=logits.device)
|
||||
history.append(argmax)
|
||||
@@ -335,7 +438,6 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
history.pop(0)
|
||||
state["step"] -= 1
|
||||
state["argmax"] = argmax
|
||||
state["rng_state"] = generator.get_state()
|
||||
finished.append((stable & confident[index]) | (state["step"] == 0))
|
||||
|
||||
# One device-to-host synchronization for the whole batch. In particular,
|
||||
@@ -348,7 +450,11 @@ class Gemma4Renoise(DllmAlgorithm):
|
||||
soft_embeds = None
|
||||
if not all(done):
|
||||
soft_embeds = self._soft_embeddings(
|
||||
probabilities.reshape(-1, self.vocab_size)
|
||||
(
|
||||
soft_probabilities
|
||||
if soft_probabilities is not None
|
||||
else probabilities
|
||||
).reshape(-1, self.vocab_size)
|
||||
).view(len(states), self.block_size, -1)
|
||||
for index in active:
|
||||
state = states[index]
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
"""dLLM FA4 compute and graph workspaces over Triton paged-cache metadata."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dllm_kv_pack import pack_prefix_current
|
||||
from sglang.kernels.ops.attention.extend_attention import extend_attention_fwd_unified
|
||||
from sglang.kernels.ops.attention.flash_attention_v4 import flash_attn_gqa_512
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.interface import (
|
||||
flash_attn_varlen_func,
|
||||
)
|
||||
from sglang.srt.layers.attention.triton_backend import (
|
||||
ForwardMetadata,
|
||||
update_sliding_window_buffer,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
get_capture_attention_variant,
|
||||
)
|
||||
from sglang.srt.utils import get_device_capability
|
||||
|
||||
|
||||
class _DenseKVWorkspace:
|
||||
def __init__(self, capacity, heads, dim, batch_size, device, dtype):
|
||||
# TMA loads can include the masked tail beyond the packed sequences.
|
||||
self.key = torch.zeros((capacity, heads, dim), device=device, dtype=dtype)
|
||||
self.value = torch.zeros_like(self.key)
|
||||
self.cu_seqlens = torch.empty(batch_size + 1, device=device, dtype=torch.int32)
|
||||
self.indices = torch.arange(capacity, device=device, dtype=torch.int64)
|
||||
self.window_start = torch.zeros(batch_size, device=device, dtype=torch.int32)
|
||||
|
||||
def fits(self, capacity, batch_size, heads, dim):
|
||||
return (
|
||||
capacity * heads * dim <= self.key.numel()
|
||||
and capacity <= self.indices.numel()
|
||||
and batch_size < self.cu_seqlens.numel()
|
||||
)
|
||||
|
||||
def view(self, capacity, batch_size, heads, dim):
|
||||
shape = (capacity, heads, dim)
|
||||
elements = capacity * heads * dim
|
||||
return (
|
||||
self.key.view(-1)[:elements].view(shape),
|
||||
self.value.view(-1)[:elements].view(shape),
|
||||
self.cu_seqlens[: batch_size + 1],
|
||||
self.indices[:capacity],
|
||||
self.window_start[:batch_size],
|
||||
)
|
||||
|
||||
|
||||
DLLM_VARLEN = "dllm_varlen"
|
||||
DLLM_FULL_WINDOW = "dllm_full_window"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DllmWindowGraphVariants:
|
||||
window_size: int
|
||||
block_size: int
|
||||
capture_labels: ClassVar[tuple[str, ...]] = (DLLM_VARLEN, DLLM_FULL_WINDOW)
|
||||
|
||||
def select(self, forward_batch: ForwardBatch, capture_batch_size: int) -> str:
|
||||
lengths = forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.batch_size > 0
|
||||
and capture_batch_size == forward_batch.batch_size
|
||||
and forward_batch.forward_mode.is_dllm_extend()
|
||||
and forward_batch.input_ids.numel()
|
||||
== forward_batch.batch_size * self.block_size
|
||||
and lengths is not None
|
||||
and int(lengths[: forward_batch.batch_size].min()) - self.block_size
|
||||
>= self.window_size
|
||||
):
|
||||
return DLLM_FULL_WINDOW
|
||||
return DLLM_VARLEN
|
||||
|
||||
|
||||
class DllmFlashAttention:
|
||||
"""FA4 computation and context-encoding graphs for DiffusionGemma."""
|
||||
|
||||
def __init__(self, backend):
|
||||
assert get_device_capability()[0] == 10, (
|
||||
"dLLM FA4 D256/D512 requires an SM100-family GPU."
|
||||
)
|
||||
self.backend = backend
|
||||
self.graph_mode = False
|
||||
self._graph_workspaces = {}
|
||||
self._eager_workspaces = {}
|
||||
self._prefill_graph_metadata = {}
|
||||
self.decode_graph_metadata = {}
|
||||
self._graph_slots = self._graph_tokens = 0
|
||||
self._prefill_capture_sizes = set()
|
||||
self._prefill_capture_max_requests = 0
|
||||
|
||||
def init_graph_state(self, max_requests, max_tokens):
|
||||
self._graph_slots = max(self._graph_slots, max_requests)
|
||||
self._graph_tokens = max(self._graph_tokens, max_tokens)
|
||||
|
||||
def init_prefill_graph_state(self, max_requests, capture_tokens):
|
||||
self.init_graph_state(max_requests, max(capture_tokens))
|
||||
self._prefill_capture_sizes = set(capture_tokens)
|
||||
self._prefill_capture_max_requests = max_requests
|
||||
|
||||
def graph_variants(self, block_size):
|
||||
window = self.backend.sliding_window_size
|
||||
if window is not None and window > 0:
|
||||
return DllmWindowGraphVariants(window, block_size)
|
||||
return None
|
||||
|
||||
def _workspace(self, key, capacity, batch_size, heads, dim, query):
|
||||
if self.graph_mode:
|
||||
capacity = self._graph_slots * key[0] + self._graph_tokens
|
||||
batch_size = self._graph_slots
|
||||
captured = self._graph_workspaces.setdefault(key, [])
|
||||
candidates = (
|
||||
captured
|
||||
if self.graph_mode
|
||||
else (w for group in self._graph_workspaces.values() for w in group)
|
||||
)
|
||||
workspace = next(
|
||||
(
|
||||
w
|
||||
for w in candidates
|
||||
if w.key.dtype == query.dtype
|
||||
and w.fits(capacity, batch_size, heads, dim)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if workspace is None:
|
||||
workspace = self._eager_workspaces.get(key)
|
||||
if workspace is None or not workspace.fits(
|
||||
capacity, batch_size, heads, dim
|
||||
):
|
||||
workspace = _DenseKVWorkspace(
|
||||
capacity, heads, dim, batch_size, query.device, query.dtype
|
||||
)
|
||||
if self.graph_mode:
|
||||
# Keep earlier captures' allocations alive when a later runner needs more capacity.
|
||||
captured.append(workspace)
|
||||
self._eager_workspaces.pop(key, None)
|
||||
else:
|
||||
self._eager_workspaces[key] = workspace
|
||||
return workspace
|
||||
|
||||
def init_forward_metadata_out_graph(self, batch, in_capture):
|
||||
self._eager_workspaces.clear()
|
||||
self.graph_mode = True
|
||||
if batch.forward_mode == ForwardMode.EXTEND:
|
||||
return self.init_prefill_metadata(batch, in_capture)
|
||||
bs = batch.batch_size
|
||||
if not in_capture:
|
||||
self.backend.forward_metadata = self.decode_graph_metadata[bs]
|
||||
self.backend._init_forward_metadata_out_graph(batch, in_capture)
|
||||
if in_capture:
|
||||
self.decode_graph_metadata[bs] = self.backend.forward_metadata
|
||||
|
||||
def init_prefill_metadata(self, batch, in_capture):
|
||||
backend = self.backend
|
||||
bs, tokens = batch.batch_size, batch.input_ids.numel()
|
||||
key = (bs, tokens)
|
||||
if in_capture:
|
||||
self.init_graph_state(bs, tokens)
|
||||
capacity = batch.max_seq_len_override or backend.max_context_len
|
||||
swa = (
|
||||
backend.sliding_window_size is not None
|
||||
and backend.sliding_window_size > 0
|
||||
)
|
||||
self._prefill_graph_metadata[key] = ForwardMetadata(
|
||||
attn_logits=None,
|
||||
attn_lse=None,
|
||||
num_kv_splits=None,
|
||||
max_extend_len=max(batch.extend_seq_lens_cpu),
|
||||
kv_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device),
|
||||
kv_indices=torch.empty(
|
||||
bs * capacity, dtype=torch.int64, device=backend.device
|
||||
),
|
||||
qo_indptr=torch.zeros(bs + 1, dtype=torch.int32, device=backend.device),
|
||||
custom_mask=None,
|
||||
mask_indptr=None,
|
||||
window_kv_indptr=torch.zeros(
|
||||
bs + 1, dtype=torch.int32, device=backend.device
|
||||
)
|
||||
if swa
|
||||
else None,
|
||||
window_kv_indices=torch.empty(
|
||||
bs * backend.sliding_window_size,
|
||||
dtype=torch.int64,
|
||||
device=backend.device,
|
||||
)
|
||||
if swa
|
||||
else None,
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=torch.empty(
|
||||
bs, dtype=torch.int64, device=backend.device
|
||||
)
|
||||
if swa
|
||||
else None,
|
||||
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.use_sliding_window_kv_pool
|
||||
else None,
|
||||
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
|
||||
if backend.kv_index_translator.is_translating
|
||||
else None,
|
||||
)
|
||||
metadata = self._prefill_graph_metadata[key]
|
||||
backend._fill_kv_indptr_and_indices(
|
||||
bs,
|
||||
batch.extend_prefix_lens,
|
||||
batch.req_pool_indices,
|
||||
metadata.kv_indices,
|
||||
kv_indptr=metadata.kv_indptr,
|
||||
)
|
||||
metadata.qo_indptr[0].zero_()
|
||||
torch.cumsum(batch.extend_seq_lens, dim=0, out=metadata.qo_indptr[1:])
|
||||
if metadata.window_kv_indices is not None:
|
||||
_, _, _, offsets = update_sliding_window_buffer(
|
||||
metadata.window_kv_indptr,
|
||||
backend.kv_index_translator,
|
||||
batch.req_pool_indices,
|
||||
backend.sliding_window_size,
|
||||
batch.extend_prefix_lens,
|
||||
bs,
|
||||
backend.device,
|
||||
backend.token_to_kv_pool,
|
||||
window_kv_indices=metadata.window_kv_indices,
|
||||
)
|
||||
metadata.window_kv_offsets.copy_(offsets)
|
||||
if metadata.swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc.copy_(
|
||||
backend.kv_index_translator.sliding_window_write_loc_for(
|
||||
batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
if metadata.out_cache_loc_full_physical is not None:
|
||||
backend.kv_index_translator.fill_capture_write_loc(
|
||||
out=metadata.out_cache_loc_full_physical,
|
||||
forward_batch=batch,
|
||||
width=metadata.out_cache_loc_full_physical.numel(),
|
||||
)
|
||||
backend.forward_metadata = metadata
|
||||
|
||||
def can_run_prefill_cuda_graph(self, batch):
|
||||
query_limit = self.get_prefill_cuda_graph_max_query_len(
|
||||
batch.input_ids.numel(), self._prefill_capture_max_requests
|
||||
)
|
||||
return (
|
||||
batch.forward_mode == ForwardMode.EXTEND
|
||||
and batch.input_ids.numel() in self._prefill_capture_sizes
|
||||
and self.backend.dcp_size == 1
|
||||
and not batch.contains_image_inputs()
|
||||
and (
|
||||
query_limit is None
|
||||
or max(batch.extend_seq_lens_cpu, default=0) <= query_limit
|
||||
)
|
||||
)
|
||||
|
||||
def get_prefill_cuda_graph_max_query_len(self, num_tokens, max_requests):
|
||||
if num_tokens <= max_requests * self.backend.dllm_block_size:
|
||||
return self.backend.dllm_block_size
|
||||
return None
|
||||
|
||||
@cache
|
||||
def _supports_layer(self, layer, dtype):
|
||||
pool = self.backend.token_to_kv_pool
|
||||
buffers = (
|
||||
pool.get_key_buffer(layer.layer_id),
|
||||
pool.get_value_buffer(layer.layer_id),
|
||||
)
|
||||
return (
|
||||
layer.qk_head_dim == layer.v_head_dim
|
||||
and layer.qk_head_dim in (256, 512)
|
||||
and dtype in (torch.float16, torch.bfloat16)
|
||||
and all(
|
||||
buffer.ndim == 3 and buffer.dtype == dtype and buffer.stride(-1) == 1
|
||||
for buffer in buffers
|
||||
)
|
||||
)
|
||||
|
||||
def forward_extend(
|
||||
self,
|
||||
layer,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
out,
|
||||
kb,
|
||||
vb,
|
||||
qo,
|
||||
ki,
|
||||
ids,
|
||||
mask,
|
||||
causal,
|
||||
mask_ptr,
|
||||
max_q,
|
||||
k_scale,
|
||||
v_scale,
|
||||
sm_scale=None,
|
||||
**options,
|
||||
):
|
||||
if (
|
||||
not self._supports_layer(layer, q.dtype)
|
||||
or mask is not None
|
||||
or options.get("sinks") is not None
|
||||
or options.get("score_mod") is not None
|
||||
or options.get("logit_cap", 0.0)
|
||||
):
|
||||
return self.backend.extend_attention_fwd(
|
||||
q,
|
||||
k.contiguous(),
|
||||
v.contiguous(),
|
||||
out,
|
||||
kb,
|
||||
vb,
|
||||
qo,
|
||||
ki,
|
||||
ids,
|
||||
mask,
|
||||
causal,
|
||||
mask_ptr,
|
||||
max_q,
|
||||
k_scale,
|
||||
v_scale,
|
||||
sm_scale=sm_scale,
|
||||
**options,
|
||||
)
|
||||
bs, heads, dim = qo.numel() - 1, k.shape[1], k.shape[2]
|
||||
is_window_layer = (
|
||||
layer.sliding_window_size is not None and layer.sliding_window_size >= 0
|
||||
)
|
||||
prefix_capacity = (
|
||||
self.backend.sliding_window_size
|
||||
if is_window_layer
|
||||
else self.backend.max_context_len
|
||||
)
|
||||
prefix_tokens = bs * prefix_capacity if self.graph_mode else ids.numel()
|
||||
capacity = prefix_tokens + q.shape[0]
|
||||
key = (prefix_capacity, heads, dim, q.dtype)
|
||||
workspace = self._workspace(key, capacity, bs, heads, dim, q)
|
||||
dense_k, dense_v, cu_seqlens, dense_ids, window_start = workspace.view(
|
||||
capacity, bs, heads, dim
|
||||
)
|
||||
pack_prefix_current(
|
||||
k,
|
||||
v,
|
||||
kb,
|
||||
vb,
|
||||
qo,
|
||||
ki,
|
||||
ids,
|
||||
dense_k,
|
||||
dense_v,
|
||||
cu_seqlens,
|
||||
prefix_capacity + max_q,
|
||||
)
|
||||
window = options.get("sliding_window_size", -1)
|
||||
if window > 0:
|
||||
prefix_lens = ki[1:] - ki[:-1]
|
||||
return extend_attention_fwd_unified(
|
||||
q,
|
||||
out,
|
||||
dense_k,
|
||||
dense_v,
|
||||
k_scale,
|
||||
v_scale,
|
||||
qo,
|
||||
cu_seqlens,
|
||||
dense_ids,
|
||||
prefix_lens,
|
||||
max_q,
|
||||
sm_scale=sm_scale,
|
||||
is_causal=causal,
|
||||
sliding_window_size=window,
|
||||
window_start_pos=window_start,
|
||||
page_size=1,
|
||||
)
|
||||
if dim == 512:
|
||||
return flash_attn_gqa_512(
|
||||
q,
|
||||
dense_k,
|
||||
dense_v,
|
||||
out,
|
||||
cu_seqlens_q=qo,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
softmax_scale=sm_scale,
|
||||
causal=causal,
|
||||
)
|
||||
if (
|
||||
is_window_layer
|
||||
and get_capture_attention_variant() == DLLM_FULL_WINDOW
|
||||
and not causal
|
||||
):
|
||||
return flash_attn_varlen_func(
|
||||
q.view(bs, -1, *q.shape[1:]),
|
||||
dense_k.view(bs, -1, heads, dim),
|
||||
dense_v.view(bs, -1, heads, dim),
|
||||
out=out.view(bs, -1, *out.shape[1:]),
|
||||
softmax_scale=sm_scale,
|
||||
causal=False,
|
||||
)
|
||||
return flash_attn_varlen_func(
|
||||
q,
|
||||
dense_k,
|
||||
dense_v,
|
||||
out=out,
|
||||
cu_seqlens_q=qo,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_q,
|
||||
max_seqlen_k=prefix_capacity + max_q,
|
||||
softmax_scale=sm_scale,
|
||||
causal=causal,
|
||||
)
|
||||
@@ -15,6 +15,7 @@ class DllmConfig:
|
||||
max_running_requests: int,
|
||||
first_done_first_out_mode: bool = False,
|
||||
requires_separate_context_encoding: bool = False,
|
||||
capture_input_preparation: bool = False,
|
||||
):
|
||||
self.algorithm = algorithm
|
||||
self.algorithm_config = algorithm_config
|
||||
@@ -23,6 +24,7 @@ class DllmConfig:
|
||||
self.max_running_requests = max_running_requests
|
||||
self.first_done_first_out_mode = first_done_first_out_mode
|
||||
self.requires_separate_context_encoding = requires_separate_context_encoding
|
||||
self.capture_input_preparation = capture_input_preparation
|
||||
|
||||
def validate_request(self, req) -> str | None:
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
@@ -113,4 +115,8 @@ class DllmConfig:
|
||||
requires_separate_context_encoding=(
|
||||
algorithm_cls.requires_separate_context_encoding
|
||||
),
|
||||
capture_input_preparation=(
|
||||
algorithm_cls.capture_input_preparation
|
||||
and algorithm_config.get("capture_input_preparation", True)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -253,6 +253,10 @@ def create_flashattention_v3_backend(runner):
|
||||
|
||||
@register_attention_backend("fa4")
|
||||
def create_flashattention_v4_backend(runner):
|
||||
if "DiffusionGemmaForBlockDiffusion" in runner.model_config.hf_config.architectures:
|
||||
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
|
||||
|
||||
return TritonAttnBackend(runner, dllm_fa4=True)
|
||||
from sglang.srt.layers.attention.flashattention_backend import (
|
||||
FlashAttentionBackend,
|
||||
)
|
||||
|
||||
@@ -159,6 +159,7 @@ class AttentionBackend(ABC):
|
||||
|
||||
# True when prefill graph metadata can use ForwardBatch.max_seq_len_override.
|
||||
supports_prefill_cuda_graph_max_context_size: bool = False
|
||||
dllm_attention = None
|
||||
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
"""Declare where this backend's scheduler-shared reads end per mode.
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_spec,
|
||||
)
|
||||
|
||||
@@ -93,6 +94,10 @@ class HybridAttnBackend(AttentionBackend):
|
||||
def supports_full_cuda_graph_chunked_prefix(self) -> bool:
|
||||
return self.prefill_backend.supports_full_cuda_graph_chunked_prefix
|
||||
|
||||
@property
|
||||
def dllm_attention(self):
|
||||
return self.prefill_backend.dllm_attention
|
||||
|
||||
@property
|
||||
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
|
||||
return self.prefill_backend.supports_prefill_cuda_graph_max_context_size
|
||||
@@ -125,12 +130,15 @@ class HybridAttnBackend(AttentionBackend):
|
||||
|
||||
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
|
||||
self.decode_backend.init_cuda_graph_state(max_bs, max_num_tokens)
|
||||
if get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill:
|
||||
# When speculative decoding is enabled, we need to initialize the backend
|
||||
# that will be used for target_verify.
|
||||
if get_exec().dllm.dllm_algorithm is not None or (
|
||||
get_spec().speculative_algorithm is not None and self.spec_attn_is_prefill
|
||||
):
|
||||
# DLLM and prefill-backed verification capture the prefill backend.
|
||||
self.prefill_backend.init_cuda_graph_state(max_bs, max_num_tokens)
|
||||
|
||||
def get_cuda_graph_seq_len_fill_value(self):
|
||||
if get_exec().dllm.dllm_algorithm is not None:
|
||||
return self.prefill_backend.get_cuda_graph_seq_len_fill_value()
|
||||
return self.decode_backend.get_cuda_graph_seq_len_fill_value()
|
||||
|
||||
def init_mha_chunk_metadata(
|
||||
@@ -159,35 +167,12 @@ class HybridAttnBackend(AttentionBackend):
|
||||
).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs)
|
||||
|
||||
def validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
self.decode_backend.validate_elastic_cuda_graph_recapture()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: Optional[torch.Tensor] = None, # For full attention
|
||||
k: Optional[torch.Tensor] = None, # For full attention
|
||||
v: Optional[torch.Tensor] = None, # For full attention
|
||||
layer: Optional[RadixAttention] = None,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
save_kv_cache: bool = True,
|
||||
*,
|
||||
mixed_qkv: Optional[torch.Tensor] = None, # For linear attention
|
||||
a: Optional[torch.Tensor] = None, # For linear attention
|
||||
b: Optional[torch.Tensor] = None, # For linear attention
|
||||
**kwargs,
|
||||
):
|
||||
"""Forward method that supports both regular attention (q, k, v) and linear attention (mixed_qkv, a, b)."""
|
||||
backend = self._select_backend(forward_batch.forward_mode)
|
||||
if mixed_qkv is not None:
|
||||
return backend.forward(
|
||||
layer=layer,
|
||||
forward_batch=forward_batch,
|
||||
save_kv_cache=save_kv_cache,
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
**kwargs,
|
||||
)
|
||||
return backend.forward(q, k, v, layer, forward_batch, save_kv_cache, **kwargs)
|
||||
backend = (
|
||||
self.prefill_backend
|
||||
if get_exec().dllm.dllm_algorithm is not None
|
||||
else self.decode_backend
|
||||
)
|
||||
backend.validate_elastic_cuda_graph_recapture()
|
||||
|
||||
def forward_decode(
|
||||
self,
|
||||
|
||||
@@ -32,6 +32,10 @@ class TboAttnBackend(AttentionBackend):
|
||||
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
|
||||
return self.primary.supports_prefill_cuda_graph_max_context_size
|
||||
|
||||
@property
|
||||
def dllm_attention(self):
|
||||
return self.primary.dllm_attention
|
||||
|
||||
@classmethod
|
||||
def init_new(cls, creator: Callable[[], AttentionBackend]):
|
||||
return cls(
|
||||
|
||||
@@ -154,6 +154,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
model_runner: ModelRunner,
|
||||
skip_prefill: bool = False,
|
||||
kv_indptr_buf: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
dllm_fa4: bool = False,
|
||||
):
|
||||
# Lazy import to avoid the initialization of cuda context
|
||||
from sglang.kernels.ops.attention.decode_attention import (
|
||||
@@ -385,7 +387,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
if not self.skip_prefill:
|
||||
self.qo_indptr = torch.zeros(
|
||||
(max_bs + 1,), dtype=torch.int64, device=model_runner.device
|
||||
(max_bs + 1,),
|
||||
dtype=torch.int32 if dllm_fa4 else torch.int64,
|
||||
device=model_runner.device,
|
||||
)
|
||||
|
||||
self.mask_indptr = torch.zeros(
|
||||
@@ -409,6 +413,11 @@ class TritonAttnBackend(AttentionBackend):
|
||||
Lq=head_dim, Lv=head_dim
|
||||
)
|
||||
self.extend_attention_block_m = block_m
|
||||
if dllm_fa4:
|
||||
from sglang.srt.dllm.attention import DllmFlashAttention
|
||||
|
||||
self.dllm_attention = DllmFlashAttention(self)
|
||||
self.supports_prefill_cuda_graph_max_context_size = True
|
||||
|
||||
def get_num_kv_splits(
|
||||
self,
|
||||
@@ -498,8 +507,10 @@ class TritonAttnBackend(AttentionBackend):
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
kv_indptr: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
kv_indptr = self.kv_indptr[: bs + 1]
|
||||
if kv_indptr is None:
|
||||
kv_indptr = self.kv_indptr[: bs + 1]
|
||||
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
self.kv_index_translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
@@ -703,6 +714,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
forward_batch: ForwardBatch,
|
||||
in_capture: bool = False,
|
||||
):
|
||||
if self.dllm_attention is not None:
|
||||
return self.dllm_attention.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture
|
||||
)
|
||||
self._init_forward_metadata_out_graph(forward_batch, in_capture)
|
||||
|
||||
def _init_forward_metadata_out_graph(self, forward_batch, in_capture):
|
||||
bs = forward_batch.batch_size
|
||||
req_pool_indices = forward_batch.req_pool_indices
|
||||
seq_lens = forward_batch.seq_lens
|
||||
@@ -810,6 +828,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Init auxiliary variables for triton attention backend."""
|
||||
|
||||
if self.dllm_attention is not None:
|
||||
self.dllm_attention.graph_mode = False
|
||||
self._dense_one_shot_kv_indptr = None
|
||||
bs = forward_batch.batch_size
|
||||
window_kv_indptr = self.window_kv_indptr
|
||||
@@ -1072,6 +1092,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
kv_indices_buf: Optional[torch.Tensor] = None,
|
||||
cuda_graph_num_kv_splits_buf: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if self.dllm_attention is not None:
|
||||
self.dllm_attention.init_graph_state(max_bs, max_num_tokens)
|
||||
self.cuda_graph_attn_logits = torch.zeros(
|
||||
(max_num_tokens, self.num_head, self.max_kv_splits, self.v_head_dim),
|
||||
dtype=torch.float32,
|
||||
@@ -1789,10 +1811,11 @@ class TritonAttnBackend(AttentionBackend):
|
||||
):
|
||||
return o
|
||||
|
||||
self.extend_attention_fwd(
|
||||
self._forward_extend_kernel(
|
||||
layer,
|
||||
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
||||
k.contiguous(),
|
||||
v.contiguous(),
|
||||
k.contiguous() if self.dllm_attention is None else k,
|
||||
v.contiguous() if self.dllm_attention is None else v,
|
||||
o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
|
||||
self.token_to_kv_pool.get_key_buffer(layer.layer_id),
|
||||
self.token_to_kv_pool.get_value_buffer(layer.layer_id),
|
||||
@@ -1818,6 +1841,11 @@ class TritonAttnBackend(AttentionBackend):
|
||||
)
|
||||
return o
|
||||
|
||||
def _forward_extend_kernel(self, layer, *args, **kwargs):
|
||||
if self.dllm_attention is not None:
|
||||
return self.dllm_attention.forward_extend(layer, *args, **kwargs)
|
||||
return self.extend_attention_fwd(*args, **kwargs)
|
||||
|
||||
def _dense_one_shot_kv_indptr_for(self, forward_batch: ForwardBatch):
|
||||
"""Cumulative full sequence lengths addressing the one-shot K/V rows.
|
||||
|
||||
@@ -2589,6 +2617,11 @@ def update_sliding_window_buffer(
|
||||
window_kv_indptr[-1], dtype=torch.int64, device=device
|
||||
)
|
||||
window_kv_start_idx = seq_lens - window_kv_lens
|
||||
token_mapping = (
|
||||
token_to_kv_pool.full_to_swa_index_mapping
|
||||
if isinstance(token_to_kv_pool, SWAKVPool)
|
||||
else None
|
||||
)
|
||||
translated = translator.fill_packed_read_stream(
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
seq_lens=window_kv_lens,
|
||||
@@ -2597,6 +2630,7 @@ def update_sliding_window_buffer(
|
||||
out=window_kv_indices,
|
||||
kv_start_idx=window_kv_start_idx,
|
||||
sliding_window=translator.reads_are_translated,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool):
|
||||
kv_last_index = window_kv_indptr[-1]
|
||||
|
||||
@@ -26,14 +26,14 @@ from torch import nn
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
softcap_inplace_logits as fused_softcap,
|
||||
)
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
softcap_to_float32_logits,
|
||||
)
|
||||
from sglang.srt.beam_search.logits_capture import BeamLogitsCapture
|
||||
from sglang.srt.distributed.device_communicators import triton_symm_mem_ag
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStates,
|
||||
pack_aux_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStates, pack_aux_hidden_states
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
attn_tp_all_gather,
|
||||
@@ -945,11 +945,19 @@ class LogitsProcessor(nn.Module):
|
||||
"dp_logits_scatter_returned", logits_shape=tuple(logits.shape)
|
||||
)
|
||||
|
||||
fuse_cast_softcap = (
|
||||
self.final_logit_softcapping
|
||||
and logits.is_cuda
|
||||
and logits.dtype in (torch.float16, torch.bfloat16)
|
||||
)
|
||||
logits = self._copy_logits_to_buffer(
|
||||
logits, logits_metadata, use_buffer=use_logits_buffer
|
||||
logits,
|
||||
logits_metadata,
|
||||
use_buffer=use_logits_buffer,
|
||||
softcap=self.final_logit_softcapping if fuse_cast_softcap else None,
|
||||
)
|
||||
|
||||
if self.final_logit_softcapping:
|
||||
if self.final_logit_softcapping and not fuse_cast_softcap:
|
||||
if not (_is_npu or _is_cpu):
|
||||
fused_softcap(logits, self.final_logit_softcapping)
|
||||
else:
|
||||
@@ -1148,11 +1156,19 @@ class LogitsProcessor(nn.Module):
|
||||
logits: torch.Tensor,
|
||||
logits_metadata: LogitsMetadata,
|
||||
use_buffer: bool = True,
|
||||
softcap: Optional[float] = None,
|
||||
) -> torch.Tensor:
|
||||
logits_buffer = logits_metadata.next_token_logits_buffer if use_buffer else None
|
||||
if logits.shape[-1] > self.vocab_size:
|
||||
logits = logits[:, : self.vocab_size]
|
||||
logits_width = logits.shape[-1]
|
||||
if softcap is not None:
|
||||
if logits_buffer is None or tuple(logits_buffer.shape) != tuple(
|
||||
logits.shape
|
||||
):
|
||||
logits_buffer = torch.empty_like(logits, dtype=torch.float32)
|
||||
assert logits_buffer.dtype == torch.float
|
||||
return softcap_to_float32_logits(logits, softcap, logits_buffer)
|
||||
# The shared logits buffer is keyed by vocab width and rows; skip it
|
||||
# when this batch has a different logits shape than the graph buffer.
|
||||
if logits_buffer is not None and tuple(logits_buffer.shape) == tuple(
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1
|
||||
},
|
||||
"129": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"255": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"257": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"510": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"511": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"513": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"514": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"1022": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"1023": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"1025": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"1026": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"2046": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"2047": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2049": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2050": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1
|
||||
},
|
||||
"129": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"255": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"257": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"510": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"511": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"513": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"514": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"1022": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"1023": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"1025": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": false
|
||||
},
|
||||
"1026": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"2046": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"2047": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2049": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"USE_TMA": true
|
||||
},
|
||||
"2050": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8
|
||||
}
|
||||
}
|
||||
@@ -194,11 +194,12 @@ class KVIndexTranslator:
|
||||
out: torch.Tensor,
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
sliding_window: bool = False,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
|
||||
report whether they came out translated.
|
||||
|
||||
Non-unified: the historical gather straight from ``req_to_token``.
|
||||
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
|
||||
Unified: one fused gather-and-translate, so no caller needs a
|
||||
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
|
||||
resident token, a length the pool bounds.
|
||||
@@ -206,8 +207,8 @@ class KVIndexTranslator:
|
||||
``sliding_window`` selects the swa sub-pool's own id space, built from
|
||||
VIRTUAL ids and never chained through full-physical. A ``False`` return
|
||||
means the ids are still VIRTUAL: the DCP path defers translation to
|
||||
``translate_dcp_read_ids``, and a static SWA pool maps the full ids
|
||||
through its own full->swa table.
|
||||
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
|
||||
table as ``token_mapping`` to fuse that translation into the gather.
|
||||
"""
|
||||
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
|
||||
# (the padded graph buffer), and the extra lanes have no length to bound.
|
||||
@@ -228,8 +229,9 @@ class KVIndexTranslator:
|
||||
out,
|
||||
self.req_to_token.stride(0),
|
||||
ENTRY_PAGE_SIZE=1,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return False
|
||||
return token_mapping is not None
|
||||
|
||||
if sliding_window:
|
||||
assert self._swa_v2p_table is not None, (
|
||||
|
||||
@@ -556,6 +556,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
|
||||
# For input embeddings
|
||||
input_embeds: Optional[torch.Tensor] = None
|
||||
dllm_input_preparation_state: Optional[torch.Tensor] = None
|
||||
# For token embedding overrides (sparse replacement at specific positions)
|
||||
replace_embeds: Optional[torch.Tensor] = None
|
||||
replace_positions: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -39,9 +39,7 @@ from torch.profiler import ProfilerActivity, profile
|
||||
|
||||
from sglang.srt.compilation import torch_compile_decoration
|
||||
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
graph_capture,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import (
|
||||
@@ -63,6 +61,8 @@ from sglang.srt.layers.dp_attention import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.model_executor.cuda_graph_buffer_registry import (
|
||||
CudaGraphBufferRegistry,
|
||||
GraphSlot,
|
||||
PaddingPolicy,
|
||||
build_decode_registry,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -89,12 +89,8 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor
|
||||
BreakableCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend.utils import resolve_decode_backend
|
||||
from sglang.srt.model_executor.runner_backend_utils import (
|
||||
CUDA_GRAPH_CAPTURE_FAILED_MSG,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.buffers import (
|
||||
DecodeInputBuffers,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils import CUDA_GRAPH_CAPTURE_FAILED_MSG
|
||||
from sglang.srt.model_executor.runner_utils.buffers import DecodeInputBuffers
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
_set_capture_attention_variant,
|
||||
_set_capture_lora_variant,
|
||||
@@ -108,12 +104,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.shared_read_event import make_external_event
|
||||
from sglang.srt.multiplex.pdmux_context import get_current_stream_idx, get_stream_groups
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_flags, get_parallel, get_spec
|
||||
from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
|
||||
from sglang.srt.utils import (
|
||||
empty_context,
|
||||
@@ -230,6 +221,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
pluggable self.backend that handles the actual capture/replay.
|
||||
"""
|
||||
|
||||
dllm_input_preparation = None
|
||||
dllm_attention = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
@@ -278,6 +272,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
self.dllm_config = DllmConfig.from_server_args(model_runner.server_args)
|
||||
self.is_dllm = self.dllm_config is not None
|
||||
self.dllm_input_preparation = None
|
||||
if self.is_dllm and self.dllm_config.capture_input_preparation:
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
self.dllm_input_preparation = get_algorithm_cls(
|
||||
self.dllm_config.algorithm
|
||||
).prepare_graph_inputs
|
||||
self.dllm_uses_input_embeds = (
|
||||
self.is_dllm and self.dllm_config.requires_separate_context_encoding
|
||||
)
|
||||
@@ -309,13 +310,22 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
elif self.is_dllm:
|
||||
self.capture_forward_mode = ForwardMode.DLLM_EXTEND
|
||||
|
||||
self.dllm_attention = self.attn_backend.dllm_attention
|
||||
self.attention_graph_variants: Optional[AttentionGraphVariants] = (
|
||||
create_attention_graph_variants(model_runner.model_config.hf_config)
|
||||
self.dllm_attention.graph_variants(self.captured_req_width)
|
||||
if self.dllm_attention is not None and self.is_dllm
|
||||
else create_attention_graph_variants(model_runner.model_config.hf_config)
|
||||
or create_dsv41_candidate_graph_variants(
|
||||
model_runner, self.capture_forward_mode, self.captured_req_width
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
self.dllm_attention is not None
|
||||
and self.attention_graph_variants is not None
|
||||
):
|
||||
self._resolve_attention_variant = self.attention_graph_variants.select
|
||||
|
||||
# --- bucket sizes ---------------------------------------------
|
||||
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(
|
||||
model_runner, self.captured_req_width
|
||||
@@ -461,6 +471,17 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
num_dp_ranks=self.num_dp_ranks,
|
||||
source=self.buffers,
|
||||
)
|
||||
if self.dllm_input_preparation is not None:
|
||||
width = self.buffers.input_embeds.shape[-1]
|
||||
self.buffer_registry.register_slot(
|
||||
GraphSlot(
|
||||
"dllm_input_preparation_state",
|
||||
lambda bs, tokens: (tokens, width),
|
||||
self.buffers.input_embeds.dtype,
|
||||
axis="tokens",
|
||||
padding_policy=PaddingPolicy.ZERO,
|
||||
)
|
||||
)
|
||||
|
||||
# Captures the per-replay attention-metadata prep into a small CUDA
|
||||
# graph; see metadata_glue_graph.py for the correctness contract.
|
||||
@@ -589,9 +610,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return [num_tokens]
|
||||
return None
|
||||
|
||||
def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
|
||||
def _resolve_attention_variant(
|
||||
self, forward_batch: ForwardBatch, capture_batch_size: int
|
||||
) -> Optional[str]:
|
||||
variants = self.attention_graph_variants
|
||||
return variants.select(forward_batch) if variants is not None else None
|
||||
if variants is None:
|
||||
return None
|
||||
return variants.select(forward_batch)
|
||||
|
||||
def _resolve_lora_variant(self, forward_batch: ForwardBatch):
|
||||
if not self.record_nolora_graph:
|
||||
@@ -705,7 +730,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
stream_idx=get_current_stream_idx() if self.enable_pdmux else None,
|
||||
variant_label=self._resolve_lora_variant(forward_batch),
|
||||
attention_variant=(
|
||||
self._resolve_attention_variant(forward_batch)
|
||||
self._resolve_attention_variant(forward_batch, cuda_graph_bs)
|
||||
if self.disable_padding
|
||||
else None
|
||||
),
|
||||
@@ -1114,10 +1139,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
else [(None, None)]
|
||||
)
|
||||
variants = self.attention_graph_variants
|
||||
attention_variants = (
|
||||
variants.capture_labels if variants is not None else (None,)
|
||||
)
|
||||
for bs in capture_range:
|
||||
attention_variants = (
|
||||
variants.capture_labels if variants is not None else (None,)
|
||||
)
|
||||
if get_parallel().tp_rank == 0:
|
||||
avail_mem = get_available_gpu_memory(
|
||||
self.model_runner.device,
|
||||
@@ -1220,7 +1245,15 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
|
||||
if self.dllm_input_preparation is None:
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
|
||||
else:
|
||||
state = self.buffer_registry.get_slot(
|
||||
"dllm_input_preparation_state"
|
||||
).buffer[:num_tokens]
|
||||
kwargs["input_embeds"] = self.dllm_input_preparation(
|
||||
self.model_runner.model, forward_batch.input_ids, state
|
||||
)
|
||||
|
||||
out = forward(
|
||||
forward_batch.input_ids,
|
||||
@@ -1280,7 +1313,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
forward_batch: ForwardBatch,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
):
|
||||
if self.dllm_uses_input_embeds and forward_batch.input_embeds is None:
|
||||
if (
|
||||
self.dllm_uses_input_embeds
|
||||
and self.dllm_input_preparation is None
|
||||
and forward_batch.input_embeds is None
|
||||
):
|
||||
raise ValueError(
|
||||
"Diffusion graph replay requires prepared input embeddings"
|
||||
)
|
||||
@@ -1294,6 +1331,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.deepep_adapter.replay()
|
||||
|
||||
if not forward_batch.needs_forward_metadata_init():
|
||||
if self.dllm_input_preparation is not None:
|
||||
slot = self.buffer_registry.get_slot("dllm_input_preparation_state")
|
||||
source = forward_batch.dllm_input_preparation_state
|
||||
padded_tokens = self.bs * self.captured_req_width
|
||||
slot.buffer[: self.raw_num_token].copy_(source)
|
||||
slot.reset_padding(self.raw_num_token, padded_tokens)
|
||||
self.buffers.input_ids[self.raw_num_token : padded_tokens].zero_()
|
||||
# Pre-planned (plan-stream load_batch already ran).
|
||||
# In speculative decoding, these two fields are still needed.
|
||||
graph_size_key = (
|
||||
@@ -1339,7 +1383,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
forward_batch.input_embeds
|
||||
)
|
||||
variant_label = self._resolve_lora_variant(forward_batch)
|
||||
attention_variant = self._resolve_attention_variant(forward_batch)
|
||||
attention_variant = self._resolve_attention_variant(forward_batch, self.bs)
|
||||
stream_idx = get_current_stream_idx() if self.enable_pdmux else None
|
||||
self._replay_graph_key = self._make_graph_key(
|
||||
graph_size_key, stream_idx, variant_label, attention_variant
|
||||
@@ -1472,7 +1516,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.model_runner.hisparse_coordinator.num_real_reqs.fill_(raw_bs)
|
||||
|
||||
variant_label = self._resolve_lora_variant(forward_batch)
|
||||
attention_variant = self._resolve_attention_variant(forward_batch)
|
||||
attention_variant = self._resolve_attention_variant(forward_batch, bs)
|
||||
stream_idx = get_current_stream_idx() if self.enable_pdmux else None
|
||||
self._replay_graph_key = self._make_graph_key(
|
||||
graph_size_key, stream_idx, variant_label, attention_variant
|
||||
|
||||
@@ -288,6 +288,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
_use_draft_input_embeds = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
dllm_attention = None
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
if get_schedule().enable_mixed_chunk:
|
||||
@@ -297,8 +298,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
f"graph backend; got '{backend}'."
|
||||
)
|
||||
super().__init__(model_runner)
|
||||
self._backend_can_run_prefill_cuda_graph = getattr(
|
||||
model_runner.attn_backend, "can_run_prefill_cuda_graph", None
|
||||
self.dllm_attention = model_runner.attn_backend.dllm_attention
|
||||
self._backend_can_run_prefill_cuda_graph = (
|
||||
self.dllm_attention.can_run_prefill_cuda_graph
|
||||
if self.dllm_attention is not None
|
||||
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
|
||||
)
|
||||
# --- model flags ----------------------------------------------
|
||||
self.quant_config = getattr(model_runner.model, "quant_config", None)
|
||||
@@ -481,6 +485,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
max_req = prefill_config.full_prefill_max_req
|
||||
assert max_req is not None, "full_prefill_max_req must be resolved"
|
||||
self._capture_req_slots = max_req
|
||||
if self.dllm_attention is not None:
|
||||
self.dllm_attention.init_prefill_graph_state(
|
||||
max_req, self.capture_num_tokens
|
||||
)
|
||||
|
||||
# BCG/Full record LoRA kernels, so the metadata they read must live in
|
||||
# static buffers refreshed in place per batch; unsupported LoRA
|
||||
@@ -505,7 +513,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# This flag controls whether the model dispatches through the distinct
|
||||
# chunked-prefix topology; backend capability is validated separately.
|
||||
self._capture_chunked_prefix = (
|
||||
self._is_full_backend and not get_schedule().disable_chunked_prefix_cache
|
||||
self._is_full_backend
|
||||
and not get_schedule().disable_chunked_prefix_cache
|
||||
and self.dllm_attention is None
|
||||
)
|
||||
self._prefix_chunk_len = 0
|
||||
self._prefix_chunk_capacity = 0
|
||||
@@ -1406,6 +1416,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
context_length = min(
|
||||
self.max_context_size or model_context_length, model_context_length
|
||||
)
|
||||
if self.dllm_attention is not None and self._is_full_backend:
|
||||
query_limit = self.dllm_attention.get_prefill_cuda_graph_max_query_len(
|
||||
num_tokens, self._capture_req_slots
|
||||
)
|
||||
if query_limit is not None:
|
||||
context_length = min(context_length, query_limit)
|
||||
# A prefill bucket is an aggregate token count. Capture it as the
|
||||
# fewest synthetic requests, with every request containing no more
|
||||
# than context_length tokens.
|
||||
@@ -2153,7 +2169,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
def _finalize_execute_output(
|
||||
self, output
|
||||
) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]:
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
if output is None:
|
||||
return None
|
||||
if isinstance(output, LogitsProcessorOutput):
|
||||
return self._trim_logits_output(output)
|
||||
if isinstance(output, EmbeddingPoolerOutput):
|
||||
@@ -2170,7 +2188,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
def execute(
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]:
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
self._validate_capture_hidden_mode(forward_batch)
|
||||
with self.backend.replay_session():
|
||||
static_forward_batch = self.load_batch(forward_batch, **kwargs)
|
||||
|
||||
@@ -21,7 +21,13 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from sglang.kernels.ops.attention.gemma_qkv_norm_rope import gemma_qkv_norm_rope
|
||||
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
|
||||
from sglang.kernels.ops.layernorm.rmsnorm_fanout import (
|
||||
can_use_rmsnorm_fanout,
|
||||
rmsnorm_fanout,
|
||||
)
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.activation import GeluAndMul
|
||||
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -57,7 +63,7 @@ from sglang.srt.models.gemma4_mm import (
|
||||
)
|
||||
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils import add_prefix, is_sm100_supported, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -162,7 +168,28 @@ class DiffusionGemmaAttention(nn.Module):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
if q.is_cuda:
|
||||
fuse_rope = (
|
||||
q.is_cuda
|
||||
and q.dtype in (torch.bfloat16, torch.float16)
|
||||
and 0 < q.shape[0] <= 1024
|
||||
and is_sm100_supported()
|
||||
)
|
||||
if fuse_rope:
|
||||
gemma_qkv_norm_rope(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self.rotary_emb.rotary_dim,
|
||||
self.q_norm.eps,
|
||||
)
|
||||
elif q.is_cuda:
|
||||
gemma_qkv_rmsnorm(
|
||||
q,
|
||||
k,
|
||||
@@ -185,7 +212,8 @@ class DiffusionGemmaAttention(nn.Module):
|
||||
v.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
).flatten(-2, -1)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
if not fuse_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
attn = (
|
||||
self.attn
|
||||
@@ -289,6 +317,7 @@ class DiffusionGemmaDecoderLayer(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
self.use_fanout = is_sm100_supported() and not is_batch_invariant_mode_enabled()
|
||||
eps = config.rms_norm_eps
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
@@ -310,14 +339,30 @@ class DiffusionGemmaDecoderLayer(nn.Module):
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
h1 = self.post_feedforward_layernorm_1(
|
||||
self.mlp(self.pre_feedforward_layernorm(residual))
|
||||
)
|
||||
use_fanout = self.use_fanout and can_use_rmsnorm_fanout(residual)
|
||||
if use_fanout:
|
||||
if not self.router._scale_fused:
|
||||
self.router.fuse_scale()
|
||||
dense_in, moe_in, router_in = rmsnorm_fanout(
|
||||
residual,
|
||||
self.pre_feedforward_layernorm.weight,
|
||||
self.pre_feedforward_layernorm_2.weight,
|
||||
self.router.norm.weight,
|
||||
self.pre_feedforward_layernorm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
dense_in = self.pre_feedforward_layernorm(residual)
|
||||
h1 = self.post_feedforward_layernorm_1(self.mlp(dense_in))
|
||||
del dense_in
|
||||
if use_fanout:
|
||||
router_logits, _ = self.router.proj(router_in)
|
||||
del router_in
|
||||
else:
|
||||
moe_in = self.pre_feedforward_layernorm_2(residual)
|
||||
router_logits = self.router(residual)
|
||||
# MoE branch: the router gates on the RAW residual (the router's input
|
||||
# norm is with_scale=False, so ln2's learned per-channel weights would
|
||||
# change the routed direction). Experts still get the ln2-normed input.
|
||||
moe_in = self.pre_feedforward_layernorm_2(residual)
|
||||
router_logits = self.router(residual)
|
||||
h2 = self.post_feedforward_layernorm_2(self.moe(moe_in, router_logits))
|
||||
|
||||
hidden_states = self.post_feedforward_layernorm(h1 + h2)
|
||||
@@ -479,6 +524,10 @@ class DiffusionGemmaForBlockDiffusion(PreTrainedModel):
|
||||
**kwargs,
|
||||
):
|
||||
is_context_encoding = not forward_batch.forward_mode.is_dllm_extend()
|
||||
if not is_context_encoding and input_embeds is None:
|
||||
input_embeds = self.prepare_dllm_input_embeds(
|
||||
input_ids, forward_batch.dllm_input_preparation_state
|
||||
)
|
||||
if is_context_encoding and forward_batch.contains_image_inputs():
|
||||
Gemma4ForConditionalGeneration.prepare_attn_masks(
|
||||
self, forward_batch, input_ids, torch.bool
|
||||
|
||||
Reference in New Issue
Block a user