[Diffusion] Fuse lossless LingBot World FP32 normalization (#40425)

This commit is contained in:
Xiaoyu Zhang
2026-09-24 18:41:10 +08:00
committed by GitHub
parent d94d784441
commit 26c63335e2
6 changed files with 516 additions and 30 deletions
@@ -191,6 +191,7 @@ def _layernorm_modulate_kernel(
STORE_BF16: tl.constexpr,
QUANTIZE_FP8: tl.constexpr,
HAS_SHIFT: tl.constexpr = True,
POST_OP: tl.constexpr = "bf16_modulate",
):
pid = tl.program_id(0).to(tl.int64)
row_offs = pid * ROWS + tl.arange(0, ROWS)
@@ -256,21 +257,37 @@ def _layernorm_modulate_kernel(
mask=mask,
other=0.0,
).to(tl.float32)
y = round_bf16_to_fp32(rstd * (x - mean))
y = rstd * (x - mean)
if POST_OP == "bf16_modulate":
y = round_bf16_to_fp32(y)
sc = tl.load(
scale_ptr + batch[:, None] * scale_row_stride + cols[None, :],
mask=mask,
other=0.0,
).to(tl.float32)
one_plus = round_bf16_to_fp32(1.0 + sc)
y = round_bf16_to_fp32(y * one_plus)
if HAS_SHIFT:
if POST_OP == "bf16_modulate":
one_plus = round_bf16_to_fp32(1.0 + sc)
y = round_bf16_to_fp32(y * one_plus)
if HAS_SHIFT:
sh = tl.load(
shift_ptr + batch[:, None] * scale_row_stride + cols[None, :],
mask=mask,
other=0.0,
).to(tl.float32)
y = y + sh
else:
sh = tl.load(
shift_ptr + batch[:, None] * scale_row_stride + cols[None, :],
mask=mask,
other=0.0,
).to(tl.float32)
y = y + sh
if POST_OP == "fp32_affine":
# Native FP32 LayerNorm contracts gamma * normalized + beta.
y = tl.fma(y, sc, sh)
else:
# Native FP32 LN, separate FP32 multiply/add, one BF16 store.
# The caller disables implicit FMA contraction.
y = y * (1.0 + sc) + sh
if STORE_BF16:
tl.store(y_ptr + row_base[:, None] + cols[None, :], y, mask=mask)
if QUANTIZE_FP8:
@@ -420,6 +437,66 @@ def _fake_ln_modulate(
return torch.empty_like(x)
def try_fused_fp32_layernorm_bf16(
x: torch.Tensor,
scale: torch.Tensor,
shift: torch.Tensor,
eps: float,
*,
affine: bool = False,
) -> torch.Tensor | None:
"""BF16 input/output with native FP32 LN and FP32 affine or adaLN math.
Replicate ``layer_norm(x.float()).bfloat16()`` with FP32 affine rows,
or affine-free FP32 LN followed by ``* (1 + scale) + shift``. Callers
must verify the live Torch reduction dispatch before enabling this path.
"""
if (
not _is_bf16_cuda(x)
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or x.ndim != 3
or x.numel() == 0
or not x.is_contiguous()
or x.shape[-1] % 4 != 0
or x.shape[-1] > 8192
or scale.dtype != torch.float32
or shift.dtype != torch.float32
or scale.device != x.device
or shift.device != x.device
):
return None
batch, tokens, channels = x.shape
stride = _mod_row_stride(scale, batch, channels)
if stride is None or _mod_row_stride(shift, batch, channels) != stride:
return None
output = torch.empty_like(x)
rows = 2
_layernorm_modulate_kernel[(triton.cdiv(batch * tokens, rows),)](
output,
output,
x,
scale,
shift,
scale,
tokens,
batch * tokens,
stride,
eps,
D=channels,
ROWS=rows,
FP8_DTYPE=fp8_dtype_to_triton(fp8_dtype),
FP8_MIN=fp8_min,
FP8_MAX=fp8_max,
STORE_BF16=True,
QUANTIZE_FP8=False,
POST_OP="fp32_affine" if affine else "fp32_modulate",
num_warps=4 if channels >= 2048 else 2,
enable_fp_fusion=False,
)
return output
def fused_layernorm_modulate_raw(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None, eps: float
) -> torch.Tensor:
@@ -607,6 +607,8 @@ _EXPORTS: dict[str, str] = {
"fuse_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
"fuse_residual_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
"fuse_scale_shift_kernel": "modulate.scale_shift_triton",
"try_fused_fp32_layernorm_bf16": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
"timestep_embedding": "modulate.timestep_embedding_jit",
"can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
@@ -279,7 +279,7 @@ def _fused_scale_shift_4d_kernel(
normalized_ptr,
scale_ptr,
shift_ptr,
scale_constant: tl.constexpr, # scale_constant is either 0 or 1.
scale_constant: tl.constexpr, # None omits the constant addition.
inner_dim,
seq_len,
num_frames,
@@ -311,8 +311,13 @@ def _fused_scale_shift_4d_kernel(
scale = tl.load(scale_ptrs, mask=mask, other=0.0)
shift = tl.load(shift_ptrs, mask=mask, other=0.0)
scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype)
output = normalized * (scale_const_tensor + scale) + shift
if scale_constant is None:
# CuTe's residual path has no extra +0 on the gate. In particular,
# adding +0 would change a negative-zero gate before multiplication.
output = normalized * scale + shift
else:
scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype)
output = normalized * (scale_const_tensor + scale) + shift
tl.store(out_ptrs, output, mask=mask)
@@ -385,6 +390,51 @@ def fuse_scale_shift_kernel_blc_opt(
tl.store(y_ptr + x_off, y, mask=mask)
def try_fused_scaled_residual_bf16(
residual: torch.Tensor, x: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor | None:
"""Return CuTe's BF16 residual output without computing its unused norm."""
if (
not is_cuda()
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or not x.is_cuda
or x.dtype != torch.bfloat16
or residual.dtype != x.dtype
or gate.dtype != torch.float32
or residual.device != x.device
or gate.device != x.device
or x.ndim != 3
or residual.shape != x.shape
or gate.ndim != 4
or gate.shape[0] != x.shape[0]
or gate.shape[2:] != (1, x.shape[-1])
or gate.shape[1] == 0
or x.shape[1] % gate.shape[1] != 0
or not x.is_contiguous()
or x.numel() == 0
):
return None
batch, tokens, channels = x.shape
frames = gate.shape[1]
output = torch.empty_like(x)
block_n = max(64, min(512, triton.next_power_of_2(channels)))
_fused_scale_shift_4d_kernel[(batch * tokens, triton.cdiv(channels, block_n))](
output,
x,
gate.reshape(batch * frames, channels).contiguous(),
residual.contiguous(),
None,
channels,
tokens,
frames,
tokens // frames,
BLOCK_N=block_n,
num_warps=2 if block_n == 64 else 4,
)
return output
def fuse_scale_shift_kernel(
x: torch.Tensor,
scale: torch.Tensor,
@@ -12,6 +12,13 @@ import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_modulate_scale_shift_cuda,
modulate_scale_shift,
try_fused_fp32_layernorm_bf16,
try_fused_scaled_residual_bf16,
)
from sglang.multimodal_gen.configs.models.dits import LingBotWorldVideoConfig
from sglang.multimodal_gen.runtime.distributed import (
divide,
@@ -196,6 +203,24 @@ class LingBotWorldCamConditioner(nn.Module):
if scale_shift is None:
scale_shift = self.compute_scale_shift(c2ws_plucker_emb)
cam_scale, cam_shift = scale_shift
if (
hidden_states.is_cuda
and torch.version.hip is None
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
and hidden_states.is_contiguous()
and cam_scale.is_contiguous()
and cam_shift.is_contiguous()
and cam_scale.shape == cam_shift.shape == hidden_states.shape
):
channels = hidden_states.shape[-1]
x = hidden_states.view(-1, 1, channels)
scale = cam_scale.view(-1, channels)
shift = cam_shift.view(-1, channels)
if can_use_modulate_scale_shift_cuda(x, scale, shift):
# This existing CUDA kernel preserves every storage-dtype
# rounding boundary and already owns its runtime fallback.
return modulate_scale_shift(x, scale, shift).view_as(hidden_states)
return (1.0 + cam_scale) * hidden_states + cam_shift
@@ -909,6 +934,165 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
self.cam_conditioner = LingBotWorldCamConditioner(self.hidden_dim)
self._fused_qkv_weight = None
self._fused_qkv_bias = None
self._norm1_modulation_gate = BitExactFusionGate(
"LingBot norm1 modulation", per_signature=True
)
self._cross_norm_gate = BitExactFusionGate(
"LingBot cross-attention FP32 norm", per_signature=True
)
self._self_residual_gate = BitExactFusionGate(
"LingBot self-attention residual", per_signature=True
)
def _fp32_norm(
self,
hidden_states: torch.Tensor,
scale: torch.Tensor | None = None,
shift: torch.Tensor | None = None,
) -> torch.Tensor:
modulate = scale is not None
norm = self.norm1 if modulate else self.self_attn_residual_norm.norm
gate = self._norm1_modulation_gate if modulate else self._cross_norm_gate
batch, tokens, channels = hidden_states.shape
frames = scale.shape[1] if modulate else 1
def reference():
if not modulate:
return norm(hidden_states)
return (
(
norm(hidden_states.float()).unflatten(1, (frames, tokens // frames))
* (1 + scale)
+ shift
)
.flatten(1, 2)
.to(hidden_states.dtype)
)
if (
hidden_states.is_cuda
and torch.version.hip is None
# The FP32 Welford order is validated against Hopper's native
# LayerNorm. A matching first input on B200 does not establish
# equivalence for later inputs or graph replays.
and current_platform.is_hopper()
and hidden_states.dtype == torch.bfloat16
and hidden_states.is_contiguous()
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
and not gate.disabled
and frames > 0
and tokens % frames == 0
and (
norm.weight is None and norm.bias is None
if modulate
else norm.weight is not None and norm.bias is not None
)
):
sig = (hidden_states.device, hidden_states.shape, frames, norm.eps)
verified = gate.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
try:
if modulate:
affine_scale = scale.reshape(
batch * frames, channels
).contiguous()
affine_shift = shift.reshape(
batch * frames, channels
).contiguous()
else:
affine_scale = (
norm._cached_fp32_param(
"_weight_fp32_cache", norm.weight, hidden_states.device
)
.view(1, channels)
.expand(batch, -1)
)
affine_shift = (
norm._cached_fp32_param(
"_bias_fp32_cache", norm.bias, hidden_states.device
)
.view(1, channels)
.expand(batch, -1)
)
out = try_fused_fp32_layernorm_bf16(
hidden_states.view(batch * frames, tokens // frames, channels),
affine_scale,
affine_shift,
norm.eps,
affine=not modulate,
)
if out is not None:
out = out.view_as(hidden_states)
if verified:
return out
return gate.accept_or_fallback(
out,
reference(),
sig=sig,
equal=lambda a, b: torch.equal(
a.view(torch.int16), b.view(torch.int16)
),
logger=logger,
)
except Exception as exc:
gate.on_exception(exc, logger=logger)
return reference()
def _self_attn_residual(
self,
hidden_states: torch.Tensor,
attn_output: torch.Tensor,
gate_msa: torch.Tensor,
) -> torch.Tensor:
def reference():
zero = hidden_states.new_zeros((1,))
return self.self_attn_residual_norm(
hidden_states, attn_output, gate_msa, zero, zero
)[1]
gate = self._self_residual_gate
if (
hidden_states.is_cuda
and torch.version.hip is None
and hidden_states.dtype == attn_output.dtype == torch.bfloat16
and gate_msa.dtype == torch.float32
and hidden_states.shape[-1] % 256 == 0
and hidden_states.shape[-1] <= 8192
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
and not gate.disabled
):
sig = (
hidden_states.device,
hidden_states.shape,
hidden_states.stride(),
attn_output.stride(),
gate_msa.shape,
gate_msa.stride(),
)
verified = gate.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
try:
out = try_fused_scaled_residual_bf16(
hidden_states, attn_output.contiguous(), gate_msa
)
if out is None:
return reference()
if verified:
return out
return gate.accept_or_fallback(
out,
reference(),
sig=sig,
equal=lambda a, b: torch.equal(
a.view(torch.int16), b.view(torch.int16)
),
logger=logger,
)
except Exception as exc:
gate.on_exception(exc, logger=logger)
return reference()
def _can_fuse_qkv_projection(self) -> bool:
if self._fused_qkv_weight is not None:
@@ -1057,24 +1241,12 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
) -> torch.Tensor:
if hidden_states.dim() == 4:
hidden_states = hidden_states.squeeze(1)
num_frames = temb.shape[1]
seqlen_per_frame = hidden_states.shape[1] // num_frames
orig_dtype = hidden_states.dtype
e = self.scale_shift_table + temb.float()
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(
6, dim=2
)
norm_hidden_states = (
(
self.norm1(hidden_states.float()).unflatten(
dim=1, sizes=(num_frames, seqlen_per_frame)
)
* (1 + scale_msa)
+ shift_msa
)
.flatten(1, 2)
.to(orig_dtype)
)
norm_hidden_states = self._fp32_norm(hidden_states, scale_msa, shift_msa)
query, key, value = self._project_qkv(norm_hidden_states)
if self.tp_rmsnorm:
query = tensor_parallel_rms_norm(query, self.norm_q)
@@ -1103,12 +1275,9 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
attn_output, _ = self.to_out(attn_output)
attn_output = attn_output.squeeze(1)
residual_zero = torch.zeros(
(1,), device=hidden_states.device, dtype=hidden_states.dtype
)
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
hidden_states, attn_output, gate_msa, residual_zero, residual_zero
)
# Camera conditioning changes the residual before cross-attention LN;
# the normalized output of the earlier fused residual+LN was discarded.
hidden_states = self._self_attn_residual(hidden_states, attn_output, gate_msa)
hidden_states = self.cam_conditioner(
hidden_states.to(orig_dtype),
c2ws_plucker_emb,
@@ -1118,9 +1287,7 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
else self._cam_conditioner_scale_shift(c2ws_plucker_emb)
),
)
norm_hidden_states = self.self_attn_residual_norm.norm(hidden_states).to(
orig_dtype
)
norm_hidden_states = self._fp32_norm(hidden_states).to(orig_dtype)
attn_output = self._cross_attn_with_cache(
norm_hidden_states, encoder_hidden_states, crossattn_cache