From 26c63335e2296100e47e6dca042320951ef69a06 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Thu, 24 Sep 2026 18:41:10 +0800 Subject: [PATCH] [Diffusion] Fuse lossless LingBot World FP32 normalization (#40425) --- docs/docs/sglang-diffusion/fused_kernels.mdx | 8 + .../kda_kernels/layernorm_modulate_triton.py | 87 +++++++- .../sglang/kernels/ops/diffusion/__init__.py | 2 + .../diffusion/modulate/scale_shift_triton.py | 56 ++++- .../runtime/models/dits/lingbot_world.py | 211 ++++++++++++++++-- .../ops/diffusion/test_lingbot_modulation.py | 182 +++++++++++++++ 6 files changed, 516 insertions(+), 30 deletions(-) create mode 100644 test/registered/kernels/ops/diffusion/test_lingbot_modulation.py diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index e8edf5e319..1af56f971a 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -105,6 +105,7 @@ These fusion families mount under both `quality="extra-high"` and | `scale_residual_norm_scale_shift` | KDA / Triton / CuTe-DSL / FlyDSL | bit-exact (Triton) or backend-specific close contract | the above plus the preceding `residual + gate * update` | | `scale_residual_norm_scale_shift_nvfp4` | JIT CUDA | matches the selected NVFP4 producer contract | Qwen residual LayerNorm/modulation + FC1 NVFP4 quantization | | `layernorm_modulate` | Triton | bit-exact | affine-free LayerNorm + adaLN modulate | +| `try_fused_fp32_layernorm_bf16` | KDA / Triton | bit-exact against the verified native dispatch | LingBot World causal FP32 LayerNorm on Hopper with FP32 affine or adaLN post-ops, including BF16 input/output conversions; other devices retain native LayerNorm | | `qk_head_layernorm` | Triton | bit-exact | per-head LayerNorm on q/k | | `qk_rmsnorm_native` | Triton | bit-exact | Z-Image per-head QK RMSNorm | | `norm_scale_shift` | KDA / CuTe-DSL / FlyDSL | backend-specific close contract | LN-or-RMS + scale/shift, many broadcast modes | @@ -120,10 +121,17 @@ These fusion families mount under both `quality="extra-high"` and | --- | --- | --- | --- | | `modulate_scale_shift` | JIT CUDA | bit-exact | `x * (1 + scale) + shift` | | `residual_gate_add` | KDA (JIT CUDA) | bit-exact | `residual + gate * update` | +| `try_fused_scaled_residual_bf16` | Triton | bit-exact against the verified native dispatch | LingBot World causal BF16 residual with an FP32 per-frame gate; skips the unused normalized output before camera conditioning | | `timestep_embedding` | JIT CUDA | close | sinusoidal timestep embedding | | `temb_table_slices` | Triton | bit-exact | see note below | | `ltx2_ada_values` | Triton | bit-exact | LTX-2 nine-way adaLN value split, slices come out contiguous | +LingBot World camera conditioning also uses `modulate_scale_shift` with one +affine row per token. The causal normalization and residual paths verify each +new signature outside CUDA graph capture and fall back to their native chains +on mismatch or unsupported layouts. These paths preserve `lossless` arithmetic; +kernel replay support does not enable BCG for a model configuration that disables it. + `temb_table_slices` is worth knowing about. The eager `(scale_shift_table + temb.float()).chunk(6, dim=2)` materializes roughly 8 GB of fp32 at 704p/121f *and* hands six **strided** slices downstream, whose `.contiguous()` calls then copy each one again. The fused kernel produces the six slices in one pass, each naturally contiguous, so the downstream copies become no-ops. diff --git a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py index e94e6c6c6d..ec4c1e32b6 100644 --- a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py +++ b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py @@ -191,6 +191,7 @@ def _layernorm_modulate_kernel( STORE_BF16: tl.constexpr, QUANTIZE_FP8: tl.constexpr, HAS_SHIFT: tl.constexpr = True, + POST_OP: tl.constexpr = "bf16_modulate", ): pid = tl.program_id(0).to(tl.int64) row_offs = pid * ROWS + tl.arange(0, ROWS) @@ -256,21 +257,37 @@ def _layernorm_modulate_kernel( mask=mask, other=0.0, ).to(tl.float32) - y = round_bf16_to_fp32(rstd * (x - mean)) + y = rstd * (x - mean) + if POST_OP == "bf16_modulate": + y = round_bf16_to_fp32(y) sc = tl.load( scale_ptr + batch[:, None] * scale_row_stride + cols[None, :], mask=mask, other=0.0, ).to(tl.float32) - one_plus = round_bf16_to_fp32(1.0 + sc) - y = round_bf16_to_fp32(y * one_plus) - if HAS_SHIFT: + if POST_OP == "bf16_modulate": + one_plus = round_bf16_to_fp32(1.0 + sc) + y = round_bf16_to_fp32(y * one_plus) + if HAS_SHIFT: + sh = tl.load( + shift_ptr + batch[:, None] * scale_row_stride + cols[None, :], + mask=mask, + other=0.0, + ).to(tl.float32) + y = y + sh + else: sh = tl.load( shift_ptr + batch[:, None] * scale_row_stride + cols[None, :], mask=mask, other=0.0, ).to(tl.float32) - y = y + sh + if POST_OP == "fp32_affine": + # Native FP32 LayerNorm contracts gamma * normalized + beta. + y = tl.fma(y, sc, sh) + else: + # Native FP32 LN, separate FP32 multiply/add, one BF16 store. + # The caller disables implicit FMA contraction. + y = y * (1.0 + sc) + sh if STORE_BF16: tl.store(y_ptr + row_base[:, None] + cols[None, :], y, mask=mask) if QUANTIZE_FP8: @@ -420,6 +437,66 @@ def _fake_ln_modulate( return torch.empty_like(x) +def try_fused_fp32_layernorm_bf16( + x: torch.Tensor, + scale: torch.Tensor, + shift: torch.Tensor, + eps: float, + *, + affine: bool = False, +) -> torch.Tensor | None: + """BF16 input/output with native FP32 LN and FP32 affine or adaLN math. + + Replicate ``layer_norm(x.float()).bfloat16()`` with FP32 affine rows, + or affine-free FP32 LN followed by ``* (1 + scale) + shift``. Callers + must verify the live Torch reduction dispatch before enabling this path. + """ + if ( + not _is_bf16_cuda(x) + or torch.is_grad_enabled() + or torch.compiler.is_compiling() + or x.ndim != 3 + or x.numel() == 0 + or not x.is_contiguous() + or x.shape[-1] % 4 != 0 + or x.shape[-1] > 8192 + or scale.dtype != torch.float32 + or shift.dtype != torch.float32 + or scale.device != x.device + or shift.device != x.device + ): + return None + batch, tokens, channels = x.shape + stride = _mod_row_stride(scale, batch, channels) + if stride is None or _mod_row_stride(shift, batch, channels) != stride: + return None + output = torch.empty_like(x) + rows = 2 + _layernorm_modulate_kernel[(triton.cdiv(batch * tokens, rows),)]( + output, + output, + x, + scale, + shift, + scale, + tokens, + batch * tokens, + stride, + eps, + D=channels, + ROWS=rows, + FP8_DTYPE=fp8_dtype_to_triton(fp8_dtype), + FP8_MIN=fp8_min, + FP8_MAX=fp8_max, + STORE_BF16=True, + QUANTIZE_FP8=False, + POST_OP="fp32_affine" if affine else "fp32_modulate", + num_warps=4 if channels >= 2048 else 2, + enable_fp_fusion=False, + ) + return output + + def fused_layernorm_modulate_raw( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None, eps: float ) -> torch.Tensor: diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 4a0758616e..fcd20a1410 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -607,6 +607,8 @@ _EXPORTS: dict[str, str] = { "fuse_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton", "fuse_residual_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton", "fuse_scale_shift_kernel": "modulate.scale_shift_triton", + "try_fused_fp32_layernorm_bf16": "sglang.kernels.kda_kernels.layernorm_modulate_triton", + "try_fused_scaled_residual_bf16": "modulate.scale_shift_triton", "try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton", "timestep_embedding": "modulate.timestep_embedding_jit", "can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton", diff --git a/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py b/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py index e791eae5e3..0399ff6de2 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py +++ b/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py @@ -279,7 +279,7 @@ def _fused_scale_shift_4d_kernel( normalized_ptr, scale_ptr, shift_ptr, - scale_constant: tl.constexpr, # scale_constant is either 0 or 1. + scale_constant: tl.constexpr, # None omits the constant addition. inner_dim, seq_len, num_frames, @@ -311,8 +311,13 @@ def _fused_scale_shift_4d_kernel( scale = tl.load(scale_ptrs, mask=mask, other=0.0) shift = tl.load(shift_ptrs, mask=mask, other=0.0) - scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype) - output = normalized * (scale_const_tensor + scale) + shift + if scale_constant is None: + # CuTe's residual path has no extra +0 on the gate. In particular, + # adding +0 would change a negative-zero gate before multiplication. + output = normalized * scale + shift + else: + scale_const_tensor = tl.full([BLOCK_N], scale_constant, dtype=scale.dtype) + output = normalized * (scale_const_tensor + scale) + shift tl.store(out_ptrs, output, mask=mask) @@ -385,6 +390,51 @@ def fuse_scale_shift_kernel_blc_opt( tl.store(y_ptr + x_off, y, mask=mask) +def try_fused_scaled_residual_bf16( + residual: torch.Tensor, x: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor | None: + """Return CuTe's BF16 residual output without computing its unused norm.""" + if ( + not is_cuda() + or torch.is_grad_enabled() + or torch.compiler.is_compiling() + or not x.is_cuda + or x.dtype != torch.bfloat16 + or residual.dtype != x.dtype + or gate.dtype != torch.float32 + or residual.device != x.device + or gate.device != x.device + or x.ndim != 3 + or residual.shape != x.shape + or gate.ndim != 4 + or gate.shape[0] != x.shape[0] + or gate.shape[2:] != (1, x.shape[-1]) + or gate.shape[1] == 0 + or x.shape[1] % gate.shape[1] != 0 + or not x.is_contiguous() + or x.numel() == 0 + ): + return None + batch, tokens, channels = x.shape + frames = gate.shape[1] + output = torch.empty_like(x) + block_n = max(64, min(512, triton.next_power_of_2(channels))) + _fused_scale_shift_4d_kernel[(batch * tokens, triton.cdiv(channels, block_n))]( + output, + x, + gate.reshape(batch * frames, channels).contiguous(), + residual.contiguous(), + None, + channels, + tokens, + frames, + tokens // frames, + BLOCK_N=block_n, + num_warps=2 if block_n == 64 else 4, + ) + return output + + def fuse_scale_shift_kernel( x: torch.Tensor, scale: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py index 0659d16cec..70bf568550 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_world.py @@ -12,6 +12,13 @@ import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F +from sglang.kernels.ops.diffusion import ( + BitExactFusionGate, + can_use_modulate_scale_shift_cuda, + modulate_scale_shift, + try_fused_fp32_layernorm_bf16, + try_fused_scaled_residual_bf16, +) from sglang.multimodal_gen.configs.models.dits import LingBotWorldVideoConfig from sglang.multimodal_gen.runtime.distributed import ( divide, @@ -196,6 +203,24 @@ class LingBotWorldCamConditioner(nn.Module): if scale_shift is None: scale_shift = self.compute_scale_shift(c2ws_plucker_emb) cam_scale, cam_shift = scale_shift + if ( + hidden_states.is_cuda + and torch.version.hip is None + and not torch.is_grad_enabled() + and not torch.compiler.is_compiling() + and hidden_states.is_contiguous() + and cam_scale.is_contiguous() + and cam_shift.is_contiguous() + and cam_scale.shape == cam_shift.shape == hidden_states.shape + ): + channels = hidden_states.shape[-1] + x = hidden_states.view(-1, 1, channels) + scale = cam_scale.view(-1, channels) + shift = cam_shift.view(-1, channels) + if can_use_modulate_scale_shift_cuda(x, scale, shift): + # This existing CUDA kernel preserves every storage-dtype + # rounding boundary and already owns its runtime fallback. + return modulate_scale_shift(x, scale, shift).view_as(hidden_states) return (1.0 + cam_scale) * hidden_states + cam_shift @@ -909,6 +934,165 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock): self.cam_conditioner = LingBotWorldCamConditioner(self.hidden_dim) self._fused_qkv_weight = None self._fused_qkv_bias = None + self._norm1_modulation_gate = BitExactFusionGate( + "LingBot norm1 modulation", per_signature=True + ) + self._cross_norm_gate = BitExactFusionGate( + "LingBot cross-attention FP32 norm", per_signature=True + ) + self._self_residual_gate = BitExactFusionGate( + "LingBot self-attention residual", per_signature=True + ) + + def _fp32_norm( + self, + hidden_states: torch.Tensor, + scale: torch.Tensor | None = None, + shift: torch.Tensor | None = None, + ) -> torch.Tensor: + modulate = scale is not None + norm = self.norm1 if modulate else self.self_attn_residual_norm.norm + gate = self._norm1_modulation_gate if modulate else self._cross_norm_gate + batch, tokens, channels = hidden_states.shape + frames = scale.shape[1] if modulate else 1 + + def reference(): + if not modulate: + return norm(hidden_states) + return ( + ( + norm(hidden_states.float()).unflatten(1, (frames, tokens // frames)) + * (1 + scale) + + shift + ) + .flatten(1, 2) + .to(hidden_states.dtype) + ) + + if ( + hidden_states.is_cuda + and torch.version.hip is None + # The FP32 Welford order is validated against Hopper's native + # LayerNorm. A matching first input on B200 does not establish + # equivalence for later inputs or graph replays. + and current_platform.is_hopper() + and hidden_states.dtype == torch.bfloat16 + and hidden_states.is_contiguous() + and not torch.is_grad_enabled() + and not torch.compiler.is_compiling() + and not gate.disabled + and frames > 0 + and tokens % frames == 0 + and ( + norm.weight is None and norm.bias is None + if modulate + else norm.weight is not None and norm.bias is not None + ) + ): + sig = (hidden_states.device, hidden_states.shape, frames, norm.eps) + verified = gate.is_verified(sig) + if verified or not torch.cuda.is_current_stream_capturing(): + try: + if modulate: + affine_scale = scale.reshape( + batch * frames, channels + ).contiguous() + affine_shift = shift.reshape( + batch * frames, channels + ).contiguous() + else: + affine_scale = ( + norm._cached_fp32_param( + "_weight_fp32_cache", norm.weight, hidden_states.device + ) + .view(1, channels) + .expand(batch, -1) + ) + affine_shift = ( + norm._cached_fp32_param( + "_bias_fp32_cache", norm.bias, hidden_states.device + ) + .view(1, channels) + .expand(batch, -1) + ) + out = try_fused_fp32_layernorm_bf16( + hidden_states.view(batch * frames, tokens // frames, channels), + affine_scale, + affine_shift, + norm.eps, + affine=not modulate, + ) + if out is not None: + out = out.view_as(hidden_states) + if verified: + return out + return gate.accept_or_fallback( + out, + reference(), + sig=sig, + equal=lambda a, b: torch.equal( + a.view(torch.int16), b.view(torch.int16) + ), + logger=logger, + ) + except Exception as exc: + gate.on_exception(exc, logger=logger) + return reference() + + def _self_attn_residual( + self, + hidden_states: torch.Tensor, + attn_output: torch.Tensor, + gate_msa: torch.Tensor, + ) -> torch.Tensor: + def reference(): + zero = hidden_states.new_zeros((1,)) + return self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, zero, zero + )[1] + + gate = self._self_residual_gate + if ( + hidden_states.is_cuda + and torch.version.hip is None + and hidden_states.dtype == attn_output.dtype == torch.bfloat16 + and gate_msa.dtype == torch.float32 + and hidden_states.shape[-1] % 256 == 0 + and hidden_states.shape[-1] <= 8192 + and not torch.is_grad_enabled() + and not torch.compiler.is_compiling() + and not gate.disabled + ): + sig = ( + hidden_states.device, + hidden_states.shape, + hidden_states.stride(), + attn_output.stride(), + gate_msa.shape, + gate_msa.stride(), + ) + verified = gate.is_verified(sig) + if verified or not torch.cuda.is_current_stream_capturing(): + try: + out = try_fused_scaled_residual_bf16( + hidden_states, attn_output.contiguous(), gate_msa + ) + if out is None: + return reference() + if verified: + return out + return gate.accept_or_fallback( + out, + reference(), + sig=sig, + equal=lambda a, b: torch.equal( + a.view(torch.int16), b.view(torch.int16) + ), + logger=logger, + ) + except Exception as exc: + gate.on_exception(exc, logger=logger) + return reference() def _can_fuse_qkv_projection(self) -> bool: if self._fused_qkv_weight is not None: @@ -1057,24 +1241,12 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock): ) -> torch.Tensor: if hidden_states.dim() == 4: hidden_states = hidden_states.squeeze(1) - num_frames = temb.shape[1] - seqlen_per_frame = hidden_states.shape[1] // num_frames orig_dtype = hidden_states.dtype e = self.scale_shift_table + temb.float() shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( 6, dim=2 ) - norm_hidden_states = ( - ( - self.norm1(hidden_states.float()).unflatten( - dim=1, sizes=(num_frames, seqlen_per_frame) - ) - * (1 + scale_msa) - + shift_msa - ) - .flatten(1, 2) - .to(orig_dtype) - ) + norm_hidden_states = self._fp32_norm(hidden_states, scale_msa, shift_msa) query, key, value = self._project_qkv(norm_hidden_states) if self.tp_rmsnorm: query = tensor_parallel_rms_norm(query, self.norm_q) @@ -1103,12 +1275,9 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock): attn_output, _ = self.to_out(attn_output) attn_output = attn_output.squeeze(1) - residual_zero = torch.zeros( - (1,), device=hidden_states.device, dtype=hidden_states.dtype - ) - norm_hidden_states, hidden_states = self.self_attn_residual_norm( - hidden_states, attn_output, gate_msa, residual_zero, residual_zero - ) + # Camera conditioning changes the residual before cross-attention LN; + # the normalized output of the earlier fused residual+LN was discarded. + hidden_states = self._self_attn_residual(hidden_states, attn_output, gate_msa) hidden_states = self.cam_conditioner( hidden_states.to(orig_dtype), c2ws_plucker_emb, @@ -1118,9 +1287,7 @@ class CausalLingBotWorldTransformerBlock(CausalWanTransformerBlock): else self._cam_conditioner_scale_shift(c2ws_plucker_emb) ), ) - norm_hidden_states = self.self_attn_residual_norm.norm(hidden_states).to( - orig_dtype - ) + norm_hidden_states = self._fp32_norm(hidden_states).to(orig_dtype) attn_output = self._cross_attn_with_cache( norm_hidden_states, encoder_hidden_states, crossattn_cache diff --git a/test/registered/kernels/ops/diffusion/test_lingbot_modulation.py b/test/registered/kernels/ops/diffusion/test_lingbot_modulation.py new file mode 100644 index 0000000000..e4617c8e1c --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_lingbot_modulation.py @@ -0,0 +1,182 @@ +"""Lossless LingBot modulation sites against their native Torch/CuTe paths.""" + +import unittest + +import torch + +from sglang.kernels.ops.diffusion import ( + BitExactFusionGate, + try_fused_fp32_layernorm_bf16, +) +from sglang.multimodal_gen.runtime.layers.layernorm import ( + FP32LayerNorm, + ScaleResidualLayerNormScaleShift, +) +from sglang.multimodal_gen.runtime.models.dits.lingbot_world import ( + CausalLingBotWorldTransformerBlock, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=35, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def make_sites(channels): + # These methods need only the existing normalization and MulAdd modules; + # avoid allocating the unrelated attention/FFN weights in kernel tests. + block = CausalLingBotWorldTransformerBlock.__new__( + CausalLingBotWorldTransformerBlock + ) + torch.nn.Module.__init__(block) + with torch.inference_mode(False): + block.norm1 = FP32LayerNorm(channels, eps=1e-6, elementwise_affine=False).cuda() + block.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + channels, eps=1e-6, elementwise_affine=True, dtype=torch.float32 + ).cuda() + block._norm1_modulation_gate = BitExactFusionGate("test norm1", per_signature=True) + block._cross_norm_gate = BitExactFusionGate("test cross norm", per_signature=True) + block._self_residual_gate = BitExactFusionGate("test residual", per_signature=True) + return block + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.version.hip is None, "NVIDIA CUDA required" +) +class TestLingBotModulation(CustomTestCase): + def assert_bits(self, actual, expected): + self.assertEqual(actual.dtype, expected.dtype) + self.assertTrue( + torch.equal(actual.view(torch.int16), expected.view(torch.int16)) + ) + + def assert_norm_dispatch(self, gate, exact=True): + # Only Hopper's native FP32 reduction order is supported. Other + # devices retain the reference without entering the exactness gate. + if torch.cuda.get_device_capability() == (9, 0): + self.assertEqual(gate.verified, exact) + self.assertEqual(gate.disabled, not exact) + else: + self.assertFalse(gate.verified) + self.assertFalse(gate.disabled) + + @torch.inference_mode() + def test_norm1_per_frame_and_replay(self): + torch.manual_seed(42) + block = make_sites(256) + x = torch.randn(2, 63, 256, device="cuda", dtype=torch.bfloat16) + table = torch.randn(2, 3, 6, 256, device="cuda", dtype=torch.float32) + shift, scale = table.chunk(6, dim=2)[:2] + normalized = block.norm1(x.float()) + expected = ( + (normalized.unflatten(1, (3, 21)) * (1 + scale) + shift) + .flatten(1, 2) + .bfloat16() + ) + raw = try_fused_fp32_layernorm_bf16( + x.view(6, 21, 256), + scale.reshape(6, 256).contiguous(), + shift.reshape(6, 256).contiguous(), + 1e-6, + ).view_as(x) + exact = torch.equal(raw.view(torch.int16), expected.view(torch.int16)) + self.assert_bits(block._fp32_norm(x, scale, shift), expected) + self.assert_norm_dispatch(block._norm1_modulation_gate, exact) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = block._fp32_norm(x, scale, shift) + x.normal_() + table.normal_() + normalized = block.norm1(x.float()) + graph.replay() + expected = ( + (normalized.unflatten(1, (3, 21)) * (1 + scale) + shift) + .flatten(1, 2) + .bfloat16() + ) + self.assert_bits(output, expected) + self.assert_norm_dispatch(block._norm1_modulation_gate) + + @torch.inference_mode() + def test_residual_matches_native_cute_and_replay(self): + for channels, frames, tokens in [(256, 1, 255), (5120, 3, 63), (5120, 1, 4680)]: + with self.subTest(channels=channels, frames=frames): + block = make_sites(channels) + residual = torch.randn( + 1, tokens, channels, device="cuda", dtype=torch.bfloat16 + ) + update = torch.randn_like(residual) + gate = torch.randn( + 1, frames, 1, channels, device="cuda", dtype=torch.float32 + ) + zero = residual.new_zeros((1,)) + expected = block.self_attn_residual_norm( + residual, update, gate, zero, zero + )[1] + self.assert_bits( + block._self_attn_residual(residual, update, gate), expected + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = block._self_attn_residual(residual, update, gate) + residual.normal_() + update.normal_() + gate.normal_() + graph.replay() + self.assert_bits( + output, + block.self_attn_residual_norm(residual, update, gate, zero, zero)[ + 1 + ], + ) + self.assertFalse(block._self_residual_gate.disabled) + # Reuse the verified signature with signed-zero gates: a +0 + # inserted before the multiply would flip all residual bits. + residual.zero_().neg_() + update.fill_(1) + gate.zero_().neg_() + self.assert_bits( + block._self_attn_residual(residual, update, gate), + block.self_attn_residual_norm(residual, update, gate, zero, zero)[ + 1 + ], + ) + block = make_sites(256) + bits = torch.arange(65536, device="cuda", dtype=torch.int32).to(torch.int16) + values = bits.view(torch.bfloat16) + update = values[torch.isfinite(values)].view(1, 255, 256) + residual = torch.zeros_like(update) + gate = torch.linspace(-2, 2, 256, device="cuda").view(1, 1, 1, 256) + zero = residual.new_zeros((1,)) + self.assert_bits( + block._self_attn_residual(residual, update, gate), + block.self_attn_residual_norm(residual, update, gate, zero, zero)[1], + ) + self.assertFalse(block._self_residual_gate.disabled) + + @torch.no_grad() + def test_affine_norm_and_live_parameters(self): + block = make_sites(5120) + x = torch.randn(1, 63, 5120, device="cuda", dtype=torch.bfloat16) + norm = block.self_attn_residual_norm.norm + norm.weight.normal_() + norm.bias.normal_() + self.assert_bits(block._fp32_norm(x), norm(x)) + self.assert_norm_dispatch(block._cross_norm_gate) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = block._fp32_norm(x) + x.normal_() + norm.weight.normal_() + norm.bias.normal_() + graph.replay() + self.assert_bits(out, norm(x)) + # Native FP32 parameter caching must also follow a BF16 conversion. + norm.bfloat16() + self.assert_bits(block._fp32_norm(x), norm(x)) + norm.weight.normal_() + self.assert_bits(block._fp32_norm(x), norm(x)) + self.assert_norm_dispatch(block._cross_norm_gate) + + +if __name__ == "__main__": + unittest.main()