[Diffusion] Fuse lossless LingBot World FP32 normalization (#40425)
This commit is contained in:
@@ -191,6 +191,7 @@ def _layernorm_modulate_kernel(
|
||||
STORE_BF16: tl.constexpr,
|
||||
QUANTIZE_FP8: tl.constexpr,
|
||||
HAS_SHIFT: tl.constexpr = True,
|
||||
POST_OP: tl.constexpr = "bf16_modulate",
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
row_offs = pid * ROWS + tl.arange(0, ROWS)
|
||||
@@ -256,21 +257,37 @@ def _layernorm_modulate_kernel(
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
y = round_bf16_to_fp32(rstd * (x - mean))
|
||||
y = rstd * (x - mean)
|
||||
if POST_OP == "bf16_modulate":
|
||||
y = round_bf16_to_fp32(y)
|
||||
sc = tl.load(
|
||||
scale_ptr + batch[:, None] * scale_row_stride + cols[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
one_plus = round_bf16_to_fp32(1.0 + sc)
|
||||
y = round_bf16_to_fp32(y * one_plus)
|
||||
if HAS_SHIFT:
|
||||
if POST_OP == "bf16_modulate":
|
||||
one_plus = round_bf16_to_fp32(1.0 + sc)
|
||||
y = round_bf16_to_fp32(y * one_plus)
|
||||
if HAS_SHIFT:
|
||||
sh = tl.load(
|
||||
shift_ptr + batch[:, None] * scale_row_stride + cols[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
y = y + sh
|
||||
else:
|
||||
sh = tl.load(
|
||||
shift_ptr + batch[:, None] * scale_row_stride + cols[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
y = y + sh
|
||||
if POST_OP == "fp32_affine":
|
||||
# Native FP32 LayerNorm contracts gamma * normalized + beta.
|
||||
y = tl.fma(y, sc, sh)
|
||||
else:
|
||||
# Native FP32 LN, separate FP32 multiply/add, one BF16 store.
|
||||
# The caller disables implicit FMA contraction.
|
||||
y = y * (1.0 + sc) + sh
|
||||
if STORE_BF16:
|
||||
tl.store(y_ptr + row_base[:, None] + cols[None, :], y, mask=mask)
|
||||
if QUANTIZE_FP8:
|
||||
@@ -420,6 +437,66 @@ def _fake_ln_modulate(
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
def try_fused_fp32_layernorm_bf16(
|
||||
x: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
eps: float,
|
||||
*,
|
||||
affine: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
"""BF16 input/output with native FP32 LN and FP32 affine or adaLN math.
|
||||
|
||||
Replicate ``layer_norm(x.float()).bfloat16()`` with FP32 affine rows,
|
||||
or affine-free FP32 LN followed by ``* (1 + scale) + shift``. Callers
|
||||
must verify the live Torch reduction dispatch before enabling this path.
|
||||
"""
|
||||
if (
|
||||
not _is_bf16_cuda(x)
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
or x.ndim != 3
|
||||
or x.numel() == 0
|
||||
or not x.is_contiguous()
|
||||
or x.shape[-1] % 4 != 0
|
||||
or x.shape[-1] > 8192
|
||||
or scale.dtype != torch.float32
|
||||
or shift.dtype != torch.float32
|
||||
or scale.device != x.device
|
||||
or shift.device != x.device
|
||||
):
|
||||
return None
|
||||
batch, tokens, channels = x.shape
|
||||
stride = _mod_row_stride(scale, batch, channels)
|
||||
if stride is None or _mod_row_stride(shift, batch, channels) != stride:
|
||||
return None
|
||||
output = torch.empty_like(x)
|
||||
rows = 2
|
||||
_layernorm_modulate_kernel[(triton.cdiv(batch * tokens, rows),)](
|
||||
output,
|
||||
output,
|
||||
x,
|
||||
scale,
|
||||
shift,
|
||||
scale,
|
||||
tokens,
|
||||
batch * tokens,
|
||||
stride,
|
||||
eps,
|
||||
D=channels,
|
||||
ROWS=rows,
|
||||
FP8_DTYPE=fp8_dtype_to_triton(fp8_dtype),
|
||||
FP8_MIN=fp8_min,
|
||||
FP8_MAX=fp8_max,
|
||||
STORE_BF16=True,
|
||||
QUANTIZE_FP8=False,
|
||||
POST_OP="fp32_affine" if affine else "fp32_modulate",
|
||||
num_warps=4 if channels >= 2048 else 2,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def fused_layernorm_modulate_raw(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None, eps: float
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -607,6 +607,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"fuse_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
|
||||
"fuse_residual_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
|
||||
"fuse_scale_shift_kernel": "modulate.scale_shift_triton",
|
||||
"try_fused_fp32_layernorm_bf16": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
||||
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
|
||||
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
|
||||
"timestep_embedding": "modulate.timestep_embedding_jit",
|
||||
"can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
|
||||
|
||||
@@ -279,7 +279,7 @@ def _fused_scale_shift_4d_kernel(
|
||||
normalized_ptr,
|
||||
scale_ptr,
|
||||
shift_ptr,
|
||||
scale_constant: tl.constexpr, # scale_constant is either 0 or 1.
|
||||
scale_constant: tl.constexpr, # None omits the constant addition.
|
||||
inner_dim,
|
||||
seq_len,
|
||||
num_frames,
|
||||
@@ -311,8 +311,13 @@ def _fused_scale_shift_4d_kernel(
|
||||
scale = tl.load(scale_ptrs, mask=mask, other=0.0)
|
||||
shift = tl.load(shift_ptrs, mask=mask, other=0.0)
|
||||
|
||||
scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype)
|
||||
output = normalized * (scale_const_tensor + scale) + shift
|
||||
if scale_constant is None:
|
||||
# CuTe's residual path has no extra +0 on the gate. In particular,
|
||||
# adding +0 would change a negative-zero gate before multiplication.
|
||||
output = normalized * scale + shift
|
||||
else:
|
||||
scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype)
|
||||
output = normalized * (scale_const_tensor + scale) + shift
|
||||
|
||||
tl.store(out_ptrs, output, mask=mask)
|
||||
|
||||
@@ -385,6 +390,51 @@ def fuse_scale_shift_kernel_blc_opt(
|
||||
tl.store(y_ptr + x_off, y, mask=mask)
|
||||
|
||||
|
||||
def try_fused_scaled_residual_bf16(
|
||||
residual: torch.Tensor, x: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor | None:
|
||||
"""Return CuTe's BF16 residual output without computing its unused norm."""
|
||||
if (
|
||||
not is_cuda()
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
or not x.is_cuda
|
||||
or x.dtype != torch.bfloat16
|
||||
or residual.dtype != x.dtype
|
||||
or gate.dtype != torch.float32
|
||||
or residual.device != x.device
|
||||
or gate.device != x.device
|
||||
or x.ndim != 3
|
||||
or residual.shape != x.shape
|
||||
or gate.ndim != 4
|
||||
or gate.shape[0] != x.shape[0]
|
||||
or gate.shape[2:] != (1, x.shape[-1])
|
||||
or gate.shape[1] == 0
|
||||
or x.shape[1] % gate.shape[1] != 0
|
||||
or not x.is_contiguous()
|
||||
or x.numel() == 0
|
||||
):
|
||||
return None
|
||||
batch, tokens, channels = x.shape
|
||||
frames = gate.shape[1]
|
||||
output = torch.empty_like(x)
|
||||
block_n = max(64, min(512, triton.next_power_of_2(channels)))
|
||||
_fused_scale_shift_4d_kernel[(batch * tokens, triton.cdiv(channels, block_n))](
|
||||
output,
|
||||
x,
|
||||
gate.reshape(batch * frames, channels).contiguous(),
|
||||
residual.contiguous(),
|
||||
None,
|
||||
channels,
|
||||
tokens,
|
||||
frames,
|
||||
tokens // frames,
|
||||
BLOCK_N=block_n,
|
||||
num_warps=2 if block_n == 64 else 4,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def fuse_scale_shift_kernel(
|
||||
x: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
|
||||
@@ -12,6 +12,13 @@ import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_modulate_scale_shift_cuda,
|
||||
modulate_scale_shift,
|
||||
try_fused_fp32_layernorm_bf16,
|
||||
try_fused_scaled_residual_bf16,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits import LingBotWorldVideoConfig
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
divide,
|
||||
@@ -196,6 +203,24 @@ class LingBotWorldCamConditioner(nn.Module):
|
||||
if scale_shift is None:
|
||||
scale_shift = self.compute_scale_shift(c2ws_plucker_emb)
|
||||
cam_scale, cam_shift = scale_shift
|
||||
if (
|
||||
hidden_states.is_cuda
|
||||
and torch.version.hip is None
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and hidden_states.is_contiguous()
|
||||
and cam_scale.is_contiguous()
|
||||
and cam_shift.is_contiguous()
|
||||
and cam_scale.shape == cam_shift.shape == hidden_states.shape
|
||||
):
|
||||
channels = hidden_states.shape[-1]
|
||||
x = hidden_states.view(-1, 1, channels)
|
||||
scale = cam_scale.view(-1, channels)
|
||||
shift = cam_shift.view(-1, channels)
|
||||
if can_use_modulate_scale_shift_cuda(x, scale, shift):
|
||||
# This existing CUDA kernel preserves every storage-dtype
|
||||
# rounding boundary and already owns its runtime fallback.
|
||||
return modulate_scale_shift(x, scale, shift).view_as(hidden_states)
|
||||
return (1.0 + cam_scale) * hidden_states + cam_shift
|
||||
|
||||
|
||||
@@ -909,6 +934,165 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
|
||||
self.cam_conditioner = LingBotWorldCamConditioner(self.hidden_dim)
|
||||
self._fused_qkv_weight = None
|
||||
self._fused_qkv_bias = None
|
||||
self._norm1_modulation_gate = BitExactFusionGate(
|
||||
"LingBot norm1 modulation", per_signature=True
|
||||
)
|
||||
self._cross_norm_gate = BitExactFusionGate(
|
||||
"LingBot cross-attention FP32 norm", per_signature=True
|
||||
)
|
||||
self._self_residual_gate = BitExactFusionGate(
|
||||
"LingBot self-attention residual", per_signature=True
|
||||
)
|
||||
|
||||
def _fp32_norm(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
scale: torch.Tensor | None = None,
|
||||
shift: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
modulate = scale is not None
|
||||
norm = self.norm1 if modulate else self.self_attn_residual_norm.norm
|
||||
gate = self._norm1_modulation_gate if modulate else self._cross_norm_gate
|
||||
batch, tokens, channels = hidden_states.shape
|
||||
frames = scale.shape[1] if modulate else 1
|
||||
|
||||
def reference():
|
||||
if not modulate:
|
||||
return norm(hidden_states)
|
||||
return (
|
||||
(
|
||||
norm(hidden_states.float()).unflatten(1, (frames, tokens // frames))
|
||||
* (1 + scale)
|
||||
+ shift
|
||||
)
|
||||
.flatten(1, 2)
|
||||
.to(hidden_states.dtype)
|
||||
)
|
||||
|
||||
if (
|
||||
hidden_states.is_cuda
|
||||
and torch.version.hip is None
|
||||
# The FP32 Welford order is validated against Hopper's native
|
||||
# LayerNorm. A matching first input on B200 does not establish
|
||||
# equivalence for later inputs or graph replays.
|
||||
and current_platform.is_hopper()
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and hidden_states.is_contiguous()
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and not gate.disabled
|
||||
and frames > 0
|
||||
and tokens % frames == 0
|
||||
and (
|
||||
norm.weight is None and norm.bias is None
|
||||
if modulate
|
||||
else norm.weight is not None and norm.bias is not None
|
||||
)
|
||||
):
|
||||
sig = (hidden_states.device, hidden_states.shape, frames, norm.eps)
|
||||
verified = gate.is_verified(sig)
|
||||
if verified or not torch.cuda.is_current_stream_capturing():
|
||||
try:
|
||||
if modulate:
|
||||
affine_scale = scale.reshape(
|
||||
batch * frames, channels
|
||||
).contiguous()
|
||||
affine_shift = shift.reshape(
|
||||
batch * frames, channels
|
||||
).contiguous()
|
||||
else:
|
||||
affine_scale = (
|
||||
norm._cached_fp32_param(
|
||||
"_weight_fp32_cache", norm.weight, hidden_states.device
|
||||
)
|
||||
.view(1, channels)
|
||||
.expand(batch, -1)
|
||||
)
|
||||
affine_shift = (
|
||||
norm._cached_fp32_param(
|
||||
"_bias_fp32_cache", norm.bias, hidden_states.device
|
||||
)
|
||||
.view(1, channels)
|
||||
.expand(batch, -1)
|
||||
)
|
||||
out = try_fused_fp32_layernorm_bf16(
|
||||
hidden_states.view(batch * frames, tokens // frames, channels),
|
||||
affine_scale,
|
||||
affine_shift,
|
||||
norm.eps,
|
||||
affine=not modulate,
|
||||
)
|
||||
if out is not None:
|
||||
out = out.view_as(hidden_states)
|
||||
if verified:
|
||||
return out
|
||||
return gate.accept_or_fallback(
|
||||
out,
|
||||
reference(),
|
||||
sig=sig,
|
||||
equal=lambda a, b: torch.equal(
|
||||
a.view(torch.int16), b.view(torch.int16)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
except Exception as exc:
|
||||
gate.on_exception(exc, logger=logger)
|
||||
return reference()
|
||||
|
||||
def _self_attn_residual(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attn_output: torch.Tensor,
|
||||
gate_msa: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
def reference():
|
||||
zero = hidden_states.new_zeros((1,))
|
||||
return self.self_attn_residual_norm(
|
||||
hidden_states, attn_output, gate_msa, zero, zero
|
||||
)[1]
|
||||
|
||||
gate = self._self_residual_gate
|
||||
if (
|
||||
hidden_states.is_cuda
|
||||
and torch.version.hip is None
|
||||
and hidden_states.dtype == attn_output.dtype == torch.bfloat16
|
||||
and gate_msa.dtype == torch.float32
|
||||
and hidden_states.shape[-1] % 256 == 0
|
||||
and hidden_states.shape[-1] <= 8192
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and not gate.disabled
|
||||
):
|
||||
sig = (
|
||||
hidden_states.device,
|
||||
hidden_states.shape,
|
||||
hidden_states.stride(),
|
||||
attn_output.stride(),
|
||||
gate_msa.shape,
|
||||
gate_msa.stride(),
|
||||
)
|
||||
verified = gate.is_verified(sig)
|
||||
if verified or not torch.cuda.is_current_stream_capturing():
|
||||
try:
|
||||
out = try_fused_scaled_residual_bf16(
|
||||
hidden_states, attn_output.contiguous(), gate_msa
|
||||
)
|
||||
if out is None:
|
||||
return reference()
|
||||
if verified:
|
||||
return out
|
||||
return gate.accept_or_fallback(
|
||||
out,
|
||||
reference(),
|
||||
sig=sig,
|
||||
equal=lambda a, b: torch.equal(
|
||||
a.view(torch.int16), b.view(torch.int16)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
except Exception as exc:
|
||||
gate.on_exception(exc, logger=logger)
|
||||
return reference()
|
||||
|
||||
def _can_fuse_qkv_projection(self) -> bool:
|
||||
if self._fused_qkv_weight is not None:
|
||||
@@ -1057,24 +1241,12 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
|
||||
) -> torch.Tensor:
|
||||
if hidden_states.dim() == 4:
|
||||
hidden_states = hidden_states.squeeze(1)
|
||||
num_frames = temb.shape[1]
|
||||
seqlen_per_frame = hidden_states.shape[1] // num_frames
|
||||
orig_dtype = hidden_states.dtype
|
||||
e = self.scale_shift_table + temb.float()
|
||||
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(
|
||||
6, dim=2
|
||||
)
|
||||
norm_hidden_states = (
|
||||
(
|
||||
self.norm1(hidden_states.float()).unflatten(
|
||||
dim=1, sizes=(num_frames, seqlen_per_frame)
|
||||
)
|
||||
* (1 + scale_msa)
|
||||
+ shift_msa
|
||||
)
|
||||
.flatten(1, 2)
|
||||
.to(orig_dtype)
|
||||
)
|
||||
norm_hidden_states = self._fp32_norm(hidden_states, scale_msa, shift_msa)
|
||||
query, key, value = self._project_qkv(norm_hidden_states)
|
||||
if self.tp_rmsnorm:
|
||||
query = tensor_parallel_rms_norm(query, self.norm_q)
|
||||
@@ -1103,12 +1275,9 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
|
||||
attn_output, _ = self.to_out(attn_output)
|
||||
attn_output = attn_output.squeeze(1)
|
||||
|
||||
residual_zero = torch.zeros(
|
||||
(1,), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
|
||||
hidden_states, attn_output, gate_msa, residual_zero, residual_zero
|
||||
)
|
||||
# Camera conditioning changes the residual before cross-attention LN;
|
||||
# the normalized output of the earlier fused residual+LN was discarded.
|
||||
hidden_states = self._self_attn_residual(hidden_states, attn_output, gate_msa)
|
||||
hidden_states = self.cam_conditioner(
|
||||
hidden_states.to(orig_dtype),
|
||||
c2ws_plucker_emb,
|
||||
@@ -1118,9 +1287,7 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock):
|
||||
else self._cam_conditioner_scale_shift(c2ws_plucker_emb)
|
||||
),
|
||||
)
|
||||
norm_hidden_states = self.self_attn_residual_norm.norm(hidden_states).to(
|
||||
orig_dtype
|
||||
)
|
||||
norm_hidden_states = self._fp32_norm(hidden_states).to(orig_dtype)
|
||||
|
||||
attn_output = self._cross_attn_with_cache(
|
||||
norm_hidden_states, encoder_hidden_states, crossattn_cache
|
||||
|
||||
Reference in New Issue
Block a user