2408 lines
94 KiB
Python
2408 lines
94 KiB
Python
"""Per-model fast paths: each model wrapper must reproduce its own reference.
|
|
|
|
A diffusion kernel is only as good as the wrapper that decides when to use it,
|
|
and that decision is model-specific: FLUX.1 and GLM-Image feed different adaLN
|
|
layouts to the same LayerNorm+modulate kernel, Sana only engages on non-default
|
|
streams, ERNIE runs a bit-exact chain unconditionally. Kernel-level numerics
|
|
live in ``test_norm.py`` / ``test_modulate.py`` / ``test_rope.py``; this file
|
|
covers the wiring: right kernel, right reference, gate ends in the right state.
|
|
|
|
Two assertion styles appear, and the difference is the point:
|
|
|
|
- ``torch.equal`` for the bit-exact default-on paths. These self-verify at
|
|
runtime, so a tolerance here would let a real regression through -- the gate
|
|
would silently fall back to eager and the fusion would simply stop running.
|
|
- a tolerance for the quality-gated paths, which are *documented* as differing
|
|
from eager at half-precision rounding-order level.
|
|
|
|
Each section also asserts the gate ended up ``verified`` / not ``disabled``:
|
|
without it a test still passes when the fast path never engaged at all.
|
|
"""
|
|
|
|
import sys
|
|
import unittest
|
|
from copy import deepcopy
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from diffusers.models.upsampling import Upsample2D
|
|
|
|
import sglang.kernels.ops.diffusion.sites.hunyuan_qknorm_site as hunyuan_qknorm
|
|
import sglang.multimodal_gen.runtime.models.dits.ernie_image as ernie_image
|
|
import sglang.multimodal_gen.runtime.models.dits.flux as flux
|
|
import sglang.multimodal_gen.runtime.models.dits.flux_2 as flux2
|
|
import sglang.multimodal_gen.runtime.models.dits.glm_image as glm_image
|
|
import sglang.multimodal_gen.runtime.models.dits.longcat_image as longcat_image
|
|
import sglang.multimodal_gen.runtime.models.dits.ltx_2 as ltx2_module
|
|
import sglang.multimodal_gen.runtime.models.dits.qwen_image as qwen_image
|
|
import sglang.multimodal_gen.runtime.models.dits.qwen_image21 as qwen_image21
|
|
import sglang.multimodal_gen.runtime.models.dits.sana as sana
|
|
from sglang.kernels.ops.diffusion import (
|
|
FUSED_LN_MODULATE,
|
|
HUNYUAN_QKNORM,
|
|
LTX2_QKNORM_SPLIT_ROPE,
|
|
LTX2_RMS_NORM_MODULATE,
|
|
QWEN_IMAGE_ADDED_QKV,
|
|
BitExactFusionGate,
|
|
can_use_wan_rmsnorm_silu,
|
|
fused_ltx2_rms_norm_modulate,
|
|
hunyuan_qkv_rope_pack,
|
|
try_flux2_token_cat_nvfp4,
|
|
wan_rmsnorm_silu,
|
|
)
|
|
from sglang.kernels.ops.diffusion.common.platform import is_cuda
|
|
from sglang.multimodal_gen.configs.models.vaes.qwenimage21 import (
|
|
QwenImage21VAEArchConfig,
|
|
QwenImage21VAEConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.models.vaes.stablediffusion3 import (
|
|
StableDiffusion3VAEConfig,
|
|
)
|
|
from sglang.multimodal_gen.runtime.layers.attention.backends import sdpa as sdpa_backend
|
|
from sglang.multimodal_gen.runtime.layers.layernorm import (
|
|
RMSNorm,
|
|
RMSNormNoWeight,
|
|
apply_qk_norm,
|
|
)
|
|
from sglang.multimodal_gen.runtime.layers.linear import MergedColumnParallelLinear
|
|
from sglang.multimodal_gen.runtime.layers.rotary_embedding.utils import (
|
|
_apply_rotary_emb,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.ernie_image import (
|
|
_ernie_gated_norm_scale_shift,
|
|
_ernie_norm_scale_shift,
|
|
_ernie_qknorm_rope,
|
|
_ernie_qknorm_rope_reference,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.flux import (
|
|
_flux_fused_ln_modulate,
|
|
_flux_norm_modulate,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.flux_2 import (
|
|
_can_use_nvfp4_swiglu_quant_fusion,
|
|
_flux2_norm_modulate,
|
|
_flux2_swiglu,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.glm_image import (
|
|
_eager_ln_modulate as _glm_eager_ln_modulate,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.glm_image import (
|
|
_glm_ln_modulate,
|
|
_glm_qk_layernorm,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.hunyuanvideo import (
|
|
_hunyuan_pack_qkv,
|
|
_hunyuan_qknorm,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.longcat_image import (
|
|
_apply_longcat_qknorm_rope,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.ltx_2 import _ltx2_rms_norm_modulate
|
|
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
|
QwenImageCrossAttention,
|
|
QwenImageTransformerBlock,
|
|
_qwen_modulation_cache_key,
|
|
_qwen_norm_out,
|
|
_split_unquantized_merged_linear,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.sana import (
|
|
_eager_ln_modulate as _sana_eager_ln_modulate,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits.sana import (
|
|
sana_ln_modulate,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes import (
|
|
autoencoder_kl_qwenimage as qwen_vae,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes import flux2_vae_cuda_opt as vae_opt
|
|
from sglang.multimodal_gen.runtime.models.vaes import (
|
|
qwen_image21_vae_cuda_opt as qwen21_vae_opt,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes import (
|
|
wan_vae_cuda_opt,
|
|
wanvae,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes.autoencoder import AutoencoderKL
|
|
from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import (
|
|
AutoencoderKLQwenImage21,
|
|
QwenImage21Resample,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes.conv_fold import (
|
|
fold_upsample2x_conv2d_weight,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path
|
|
from sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt import (
|
|
FusedWanRMSNormSiLU,
|
|
GatedChannelsLastUpsample,
|
|
VaeFastPathGate,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm
|
|
from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability
|
|
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cuda_ci(est_time=110, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
|
register_amd_ci(est_time=8, suite="nightly-amd-kernel-1-gpu", nightly=True)
|
|
|
|
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
|
|
|
# The bit-exact LayerNorm/RMSNorm fusions are NVIDIA inline PTX, so their
|
|
# guards reject on ROCm and these sites serve eager there; only the subtests
|
|
# asserting a fused outcome are CUDA-only.
|
|
requires_inline_ptx = pytest.mark.skipif(
|
|
not is_cuda(), reason="bit-exact norm fusions are NVIDIA PTX"
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _seed_cuda():
|
|
"""Every wrapper below asserts against a reference computed from the same
|
|
random draw, so the seed must be fixed per test, not per module."""
|
|
torch.cuda.manual_seed(0)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not is_cuda()
|
|
or torch.cuda.get_device_capability()[0] != 9
|
|
or sdpa_backend.torch_varlen_attn is None,
|
|
reason="packed Flash SDPA requires Hopper and PyTorch varlen attention",
|
|
)
|
|
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
|
@pytest.mark.parametrize("causal", [False, True])
|
|
@pytest.mark.parametrize(
|
|
"lengths,head_dim",
|
|
[
|
|
(([64] * 7 + [32]) * 11 + [16] * 7 + [8], 80),
|
|
([0, 64, 64, 0, 32, 0], 80),
|
|
([32] * 8, 64),
|
|
([128, 64, 128], 128),
|
|
([2048], 80),
|
|
],
|
|
)
|
|
def test_packed_sdpa_uses_native_varlen_without_changing_values(
|
|
dtype, causal, lengths, head_dim
|
|
):
|
|
bounds = [0]
|
|
for length in lengths:
|
|
bounds.append(bounds[-1] + length)
|
|
packed = torch.randn(bounds[-1], 3, 16, head_dim, device="cuda", dtype=dtype)
|
|
q, k, v = packed.unbind(1)
|
|
q, k = q.contiguous(), k.contiguous()
|
|
attention = sdpa_backend.SDPAImpl(
|
|
16, head_dim, causal=causal, softmax_scale=head_dim**-0.5
|
|
)
|
|
cu = torch.tensor(bounds, device="cuda", dtype=torch.int32)
|
|
with (
|
|
torch.no_grad(),
|
|
torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.FLASH_ATTENTION),
|
|
):
|
|
expected = torch.cat(
|
|
[
|
|
attention.forward(q[a:b][None], k[a:b][None], v[a:b][None], None)[0]
|
|
for a, b in zip(bounds[:-1], bounds[1:])
|
|
if a != b
|
|
]
|
|
)
|
|
with patch.object(attention, "forward", wraps=attention.forward) as forward:
|
|
actual = attention.forward_varlen(
|
|
q,
|
|
k,
|
|
v,
|
|
cu_seqlens=cu,
|
|
max_seqlen=max(lengths),
|
|
cu_seqlens_host=tuple(bounds),
|
|
)
|
|
assert forward.call_count == (1 if len(lengths) == 1 else 0)
|
|
assert torch.equal(actual, expected)
|
|
|
|
|
|
@pytest.mark.skipif(not is_cuda(), reason="CUDA packed SDPA fallback test")
|
|
def test_packed_sdpa_preserves_training_dropout_and_missing_api_fallbacks():
|
|
attention = sdpa_backend.SDPAImpl(4, 64, causal=True, softmax_scale=64**-0.5)
|
|
qkv = torch.randn(
|
|
64, 3, 4, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True
|
|
)
|
|
q, k, v = qkv.unbind(1)
|
|
cu = torch.tensor([0, 32, 64], device="cuda", dtype=torch.int32)
|
|
kwargs = dict(cu_seqlens=cu, max_seqlen=32, cu_seqlens_host=(0, 32, 64))
|
|
with patch.object(attention, "forward", wraps=attention.forward) as forward:
|
|
result = attention.forward_varlen(q, k, v, **kwargs)
|
|
assert forward.call_count == 2
|
|
result.float().square().mean().backward()
|
|
assert qkv.grad is not None and torch.isfinite(qkv.grad).all()
|
|
attention.dropout = 0.1
|
|
with (
|
|
torch.no_grad(),
|
|
patch.object(attention, "forward", wraps=attention.forward) as forward,
|
|
):
|
|
attention.forward_varlen(q, k, v, **kwargs)
|
|
assert forward.call_count == 2
|
|
attention.dropout = 0.0
|
|
with (
|
|
torch.no_grad(),
|
|
patch.object(sdpa_backend, "torch_varlen_attn", None),
|
|
patch.object(attention, "forward", wraps=attention.forward) as forward,
|
|
):
|
|
attention.forward_varlen(q, k, v, **kwargs)
|
|
assert forward.call_count == 2
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Qwen-Image -- final LayerNorm + adaLN scale/shift
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@requires_inline_ptx
|
|
def test_qwen_norm_out_matches_adaln_reference():
|
|
qwen_image._QWEN_NORM_OUT.disabled = False
|
|
qwen_image._QWEN_NORM_OUT.verified = False
|
|
qwen_image._QWEN_NORM_OUT_SIGS.clear()
|
|
torch.manual_seed(0)
|
|
norm_out = (
|
|
qwen_image.AdaLayerNormContinuous(
|
|
3072, 3072, elementwise_affine=False, eps=1e-6
|
|
)
|
|
.cuda()
|
|
.bfloat16()
|
|
)
|
|
hidden_states = torch.randn(1, 257, 3072, device="cuda", dtype=torch.bfloat16)
|
|
conditioning = torch.randn(1, 3072, device="cuda", dtype=torch.bfloat16)
|
|
|
|
expected = norm_out(hidden_states, conditioning)
|
|
actual = _qwen_norm_out(norm_out, hidden_states, conditioning)
|
|
|
|
assert torch.equal(actual, expected)
|
|
assert qwen_image._QWEN_NORM_OUT.verified
|
|
assert not qwen_image._QWEN_NORM_OUT.disabled
|
|
|
|
|
|
def test_qwen_norm_out_preserves_compile_path(monkeypatch):
|
|
norm_out = (
|
|
qwen_image.AdaLayerNormContinuous(16, 16, elementwise_affine=False, eps=1e-6)
|
|
.cuda()
|
|
.bfloat16()
|
|
)
|
|
hidden_states = torch.randn(1, 3, 16, device="cuda", dtype=torch.bfloat16)
|
|
conditioning = torch.randn(1, 16, device="cuda", dtype=torch.bfloat16)
|
|
expected = norm_out(hidden_states, conditioning)
|
|
|
|
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
|
monkeypatch.setattr(
|
|
qwen_image,
|
|
"fused_layernorm_modulate_raw",
|
|
lambda *args, **kwargs: pytest.fail("compile path must not dispatch kernel"),
|
|
)
|
|
|
|
assert torch.equal(_qwen_norm_out(norm_out, hidden_states, conditioning), expected)
|
|
|
|
|
|
def test_qwen_norm_out_does_not_verify_during_graph_capture(monkeypatch):
|
|
qwen_image._QWEN_NORM_OUT.disabled = False
|
|
qwen_image._QWEN_NORM_OUT.verified = False
|
|
qwen_image._QWEN_NORM_OUT_SIGS.clear()
|
|
norm_out = (
|
|
qwen_image.AdaLayerNormContinuous(
|
|
3072, 3072, elementwise_affine=False, eps=1e-6
|
|
)
|
|
.cuda()
|
|
.bfloat16()
|
|
)
|
|
hidden_states = torch.randn(1, 17, 3072, device="cuda", dtype=torch.bfloat16)
|
|
conditioning = torch.randn(1, 3072, device="cuda", dtype=torch.bfloat16)
|
|
expected = norm_out(hidden_states, conditioning)
|
|
|
|
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
|
|
monkeypatch.setattr(
|
|
qwen_image,
|
|
"fused_layernorm_modulate_raw",
|
|
lambda *args, **kwargs: pytest.fail("capture must not verify a new layout"),
|
|
)
|
|
|
|
assert torch.equal(_qwen_norm_out(norm_out, hidden_states, conditioning), expected)
|
|
assert not qwen_image._QWEN_NORM_OUT_SIGS
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# FLUX.1 -- LayerNorm + adaLN modulate, and the shared-FF GELU site
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
def _flux_eager(norm, x, scale, shift):
|
|
return norm(x) * (1 + scale[:, None]) + shift[:, None]
|
|
|
|
|
|
def _flux_site_inputs(shape, chunks, seed):
|
|
torch.manual_seed(seed)
|
|
batch, seq, hidden = shape
|
|
norm = torch.nn.LayerNorm(hidden, eps=1e-6, elementwise_affine=False).cuda()
|
|
x = (torch.randn(batch, seq, hidden, device="cuda") * 8).bfloat16()
|
|
emb = torch.randn(batch, chunks * hidden, device="cuda").bfloat16()
|
|
parts = emb.chunk(chunks, dim=1) # strided adaLN projection views
|
|
return norm, x, parts[0], parts[1]
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize(
|
|
"shape,chunks",
|
|
[
|
|
((1, 4096, 3072), 6), # dual-stream image tokens (1024^2), chunk(6)
|
|
((1, 512, 3072), 6), # dual-stream text tokens
|
|
((1, 4608, 3072), 3), # single-stream concat, chunk(3)
|
|
((2, 300, 3072), 6), # CFG batch, odd seq
|
|
],
|
|
)
|
|
def test_flux_fused_ln_modulate_is_bit_exact(shape, chunks):
|
|
# Every distinct (shape, stride, eps) signature the FLUX.1 sites emit
|
|
# must verify torch.equal on first sight and stay enabled.
|
|
norm, x, shift, scale = _flux_site_inputs(shape, chunks, seed=0)
|
|
out = _flux_fused_ln_modulate(norm, x, scale, shift)
|
|
assert out is not None
|
|
assert torch.equal(out, _flux_eager(norm, x, scale, shift))
|
|
assert not flux._FLUX_LN_MOD.disabled
|
|
assert flux._FLUX_LN_MOD.verified
|
|
|
|
|
|
@requires_inline_ptx
|
|
def test_flux_norm_modulate_bitexact_supersedes_high_fold():
|
|
# With the quality="high" affine fold mounted, the bit-exact kernel
|
|
# still takes priority, so the site output stays lossless.
|
|
site = torch.nn.Module()
|
|
FUSED_LN_MODULATE.mark(site)
|
|
assert FUSED_LN_MODULATE.mount(site)
|
|
norm, x, shift, scale = _flux_site_inputs((1, 128, 3072), 6, seed=1)
|
|
out = _flux_norm_modulate(site, norm, x, scale, shift)
|
|
assert torch.equal(out, _flux_eager(norm, x, scale, shift))
|
|
|
|
|
|
def test_flux_fused_ln_modulate_rejects_unsupported_hidden():
|
|
# hidden % 4 != 0 is outside the kernel contract and must bail out.
|
|
norm, x, shift, scale = _flux_site_inputs((1, 64, 3070), 6, seed=2)
|
|
assert _flux_fused_ln_modulate(norm, x, scale, shift) is None
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# FLUX.2 -- packed norm+modulate and packed SwiGLU views
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
|
|
class TestFlux2EagerFusions(CustomTestCase):
|
|
def setUp(self):
|
|
flux2._FLUX2_LN_MOD.disabled = False
|
|
flux2._FLUX2_LN_MOD.verified = False
|
|
flux2._FLUX2_LN_MOD_SIGS.clear()
|
|
flux2._FLUX2_SWIGLU.disabled = False
|
|
flux2._FLUX2_SWIGLU.verified = False
|
|
flux2._FLUX2_SWIGLU_SIGS.clear()
|
|
|
|
@requires_inline_ptx
|
|
def test_norm_modulate_is_bit_exact_across_sequence_lengths(self):
|
|
torch.manual_seed(0)
|
|
hidden = 256
|
|
norm = torch.nn.LayerNorm(
|
|
hidden, eps=1e-6, elementwise_affine=False, device="cuda"
|
|
)
|
|
# FLUX.2 modulation values are views of one packed projection.
|
|
params = torch.randn(1, 1, 6 * hidden, device="cuda").bfloat16()
|
|
shift, scale = params.chunk(6, dim=-1)[:2]
|
|
|
|
for seq in (17, 65):
|
|
x = torch.randn(1, seq, hidden, device="cuda").bfloat16()
|
|
expected = norm(x) * (1 + scale) + shift
|
|
actual = _flux2_norm_modulate(norm, x, scale, shift)
|
|
self.assertTrue(torch.equal(actual, expected))
|
|
|
|
self.assertFalse(flux2._FLUX2_LN_MOD.disabled)
|
|
self.assertEqual(len(flux2._FLUX2_LN_MOD_SIGS), 1)
|
|
|
|
def test_packed_swiglu_is_bit_exact_for_contiguous_and_strided_views(self):
|
|
torch.manual_seed(1)
|
|
hidden = 384
|
|
inputs = [
|
|
torch.randn(1, 19, 2 * hidden, device="cuda").bfloat16(),
|
|
torch.randn(1, 19, 3 * hidden, device="cuda").bfloat16()[..., : 2 * hidden],
|
|
]
|
|
for x in inputs:
|
|
expected = F.silu(x[..., :hidden]) * x[..., hidden:]
|
|
actual = _flux2_swiglu(x)
|
|
self.assertTrue(torch.equal(actual, expected))
|
|
|
|
self.assertFalse(flux2._FLUX2_SWIGLU.disabled)
|
|
self.assertEqual(len(flux2._FLUX2_SWIGLU_SIGS), 2)
|
|
|
|
def test_nvfp4_swiglu_quant_fusion_is_sm103_only(self):
|
|
self.assertFalse(_can_use_nvfp4_swiglu_quant_fusion(DeviceCapability(10, 0)))
|
|
self.assertTrue(_can_use_nvfp4_swiglu_quant_fusion(DeviceCapability(10, 3)))
|
|
self.assertFalse(_can_use_nvfp4_swiglu_quant_fusion(DeviceCapability(12, 0)))
|
|
|
|
def test_fp16_preserves_reference_path(self):
|
|
x = torch.randn(1, 17, 512, device="cuda", dtype=torch.float16)
|
|
expected = F.silu(x[..., :256]) * x[..., 256:]
|
|
actual = _flux2_swiglu(x)
|
|
self.assertTrue(torch.equal(actual, expected))
|
|
self.assertFalse(flux2._FLUX2_SWIGLU.disabled)
|
|
|
|
def test_packed_swiglu_rejects_non_dense_outer_stride(self):
|
|
base = torch.randn(2, 23, 512, device="cuda", dtype=torch.bfloat16)
|
|
x = base[:, :19]
|
|
self.assertNotEqual(x.stride(0), x.shape[1] * x.stride(1))
|
|
|
|
expected = F.silu(x[..., :256]) * x[..., 256:]
|
|
actual = _flux2_swiglu(x)
|
|
self.assertTrue(torch.equal(actual, expected))
|
|
self.assertEqual(len(flux2._FLUX2_SWIGLU_SIGS), 0)
|
|
|
|
def test_new_swiglu_signature_is_not_verified_during_graph_capture(self):
|
|
first = torch.randn(1, 17, 512, device="cuda", dtype=torch.bfloat16)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
_flux2_swiglu(first),
|
|
F.silu(first[..., :256]) * first[..., 256:],
|
|
)
|
|
)
|
|
self.assertEqual(len(flux2._FLUX2_SWIGLU_SIGS), 1)
|
|
|
|
second = torch.randn(1, 19, 768, device="cuda", dtype=torch.bfloat16)
|
|
with patch("torch.cuda.is_current_stream_capturing", return_value=True):
|
|
actual = _flux2_swiglu(second)
|
|
|
|
expected = F.silu(second[..., :384]) * second[..., 384:]
|
|
self.assertTrue(torch.equal(actual, expected))
|
|
self.assertEqual(len(flux2._FLUX2_SWIGLU_SIGS), 1)
|
|
|
|
@pytest.mark.skipif(
|
|
not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3),
|
|
reason="FLUX.2 token-cat NVFP4 requires Blackwell SM103",
|
|
)
|
|
def test_token_cat_nvfp4_matches_flashinfer(self):
|
|
import flashinfer
|
|
|
|
torch.manual_seed(20260830)
|
|
attention = torch.randn(1, 17, 6144, device="cuda", dtype=torch.bfloat16)
|
|
mlp = torch.randn(1, 17, 18432, device="cuda", dtype=torch.bfloat16)
|
|
global_scale = torch.tensor(0.625, device="cuda", dtype=torch.float32)
|
|
|
|
expected_fp4, expected_scales = flashinfer.fp4_quantize(
|
|
torch.cat([attention, mlp], dim=-1).view(-1, 24576), global_scale
|
|
)
|
|
actual = try_flux2_token_cat_nvfp4(attention, mlp, global_scale)
|
|
|
|
self.assertIsNotNone(actual)
|
|
actual_fp4, actual_scales = actual
|
|
self.assertTrue(torch.equal(actual_fp4, expected_fp4))
|
|
self.assertTrue(
|
|
torch.equal(
|
|
actual_scales.view(torch.uint8), expected_scales.view(torch.uint8)
|
|
)
|
|
)
|
|
|
|
def test_token_cat_nvfp4_falls_back_while_compiling(self):
|
|
attention = torch.empty(1, 1, 6144, device="cuda", dtype=torch.bfloat16)
|
|
mlp = torch.empty(1, 1, 18432, device="cuda", dtype=torch.bfloat16)
|
|
global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
|
|
|
with patch("torch.compiler.is_compiling", return_value=True):
|
|
self.assertIsNone(try_flux2_token_cat_nvfp4(attention, mlp, global_scale))
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Qwen-Image -- reuse timestep-only modulation across serial CFG branches
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
class _PackedAddedQKV(MergedColumnParallelLinear):
|
|
# The lossless split only applies to a MergedColumnParallelLinear (LoRA
|
|
# wrappers are called as they are), so the fake must be one; its __init__
|
|
# is skipped because it needs no parallel state.
|
|
def __init__(self, dim: int):
|
|
nn.Module.__init__(self)
|
|
self.output_partition_sizes = [dim, dim, dim]
|
|
self.quant_config = None
|
|
self.weight = nn.Parameter(
|
|
torch.randn(3 * dim, dim, device="cuda", dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
self.bias = nn.Parameter(
|
|
torch.randn(3 * dim, device="cuda", dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
self.calls = 0
|
|
|
|
def forward(self, x):
|
|
self.calls += 1
|
|
return F.linear(x, self.weight, self.bias), None
|
|
|
|
|
|
class _WrappedAddedQKV(nn.Module):
|
|
# Stands for a LoRA-wrapped projection: not a MergedColumnParallelLinear,
|
|
# so the lossless path must call it instead of slicing its weight.
|
|
def __init__(self, dim: int):
|
|
super().__init__()
|
|
self.weight = nn.Parameter(
|
|
torch.randn(3 * dim, dim, device="cuda", dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
self.bias = nn.Parameter(
|
|
torch.randn(3 * dim, device="cuda", dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
self.calls = 0
|
|
|
|
def forward(self, x):
|
|
self.calls += 1
|
|
return F.linear(x, self.weight, self.bias), None
|
|
|
|
|
|
def _added_qkv_attention(packed):
|
|
attention = QwenImageCrossAttention.__new__(QwenImageCrossAttention)
|
|
nn.Module.__init__(attention)
|
|
attention.use_fused_added_qkv = True
|
|
attention._unquantized_added_qkv_is_packed = True
|
|
attention.to_added_qkv = packed
|
|
QWEN_IMAGE_ADDED_QKV.mark(attention)
|
|
return attention
|
|
|
|
|
|
def test_qwen_added_qkv_lossless_calls_a_wrapped_linear_as_is():
|
|
torch.manual_seed(20260831)
|
|
dim = 64
|
|
x = torch.randn(1, 17, dim, device="cuda", dtype=torch.bfloat16)
|
|
wrapped = _WrappedAddedQKV(dim)
|
|
attention = _added_qkv_attention(wrapped)
|
|
|
|
actual = attention._get_added_qkv_projections(x)
|
|
expected = tuple(
|
|
tensor.contiguous()
|
|
for tensor in F.linear(x, wrapped.weight, wrapped.bias).chunk(3, dim=-1)
|
|
)
|
|
assert wrapped.calls == 1
|
|
assert all(torch.equal(a, e) for a, e in zip(actual, expected))
|
|
|
|
|
|
def test_qwen_added_qkv_lossless_uses_three_reference_gemms():
|
|
torch.manual_seed(20260831)
|
|
dim = 64
|
|
x = torch.randn(1, 17, dim, device="cuda", dtype=torch.bfloat16)
|
|
packed = _PackedAddedQKV(dim)
|
|
attention = _added_qkv_attention(packed)
|
|
|
|
expected_lossless = _split_unquantized_merged_linear(packed, x)
|
|
actual_lossless = attention._get_added_qkv_projections(x)
|
|
assert packed.calls == 0
|
|
assert all(
|
|
torch.equal(actual, expected)
|
|
for actual, expected in zip(actual_lossless, expected_lossless)
|
|
)
|
|
|
|
assert QWEN_IMAGE_ADDED_QKV.mount(attention)
|
|
actual_high = attention._get_added_qkv_projections(x)
|
|
expected_high = tuple(
|
|
tensor.contiguous()
|
|
for tensor in F.linear(x, packed.weight, packed.bias).chunk(3, dim=-1)
|
|
)
|
|
assert packed.calls == 1
|
|
assert all(
|
|
torch.equal(actual, expected)
|
|
for actual, expected in zip(actual_high, expected_high)
|
|
)
|
|
|
|
QWEN_IMAGE_ADDED_QKV.unmount(attention)
|
|
attention._get_added_qkv_projections(x)
|
|
assert packed.calls == 1
|
|
|
|
|
|
class _CountingProjection(nn.Module):
|
|
def __init__(self, offset: float):
|
|
super().__init__()
|
|
self.offset = offset
|
|
self.calls = 0
|
|
|
|
def forward(self, x):
|
|
self.calls += 1
|
|
return x + self.offset, None
|
|
|
|
|
|
class TestQwenImageModulationCache(CustomTestCase):
|
|
def _block(self):
|
|
block = QwenImageTransformerBlock.__new__(QwenImageTransformerBlock)
|
|
nn.Module.__init__(block)
|
|
block.img_mod = nn.ModuleList([nn.Identity(), _CountingProjection(1.0)])
|
|
block.txt_mod = nn.ModuleList([nn.Identity(), _CountingProjection(2.0)])
|
|
block._modulation_cache = None
|
|
block._shard_modulation = False
|
|
return block
|
|
|
|
def _key(self, timestep, hidden, additional_t_cond=None):
|
|
with torch.no_grad():
|
|
return _qwen_modulation_cache_key(
|
|
timestep,
|
|
additional_t_cond,
|
|
hidden,
|
|
)
|
|
|
|
def test_matching_cfg_key_reuses_both_modulation_projections(self):
|
|
block = self._block()
|
|
timestep = torch.tensor([500.0], device="cuda")
|
|
hidden = torch.empty(1, 17, 32, device="cuda", dtype=torch.bfloat16)
|
|
img_temb = torch.randn(1, 32, device="cuda", dtype=torch.bfloat16)
|
|
txt_temb = torch.randn_like(img_temb)
|
|
key = self._key(timestep, hidden)
|
|
|
|
first = block._get_modulation_params(img_temb, txt_temb, key)
|
|
second = block._get_modulation_params(img_temb, txt_temb, key)
|
|
|
|
self.assertIs(first[0], second[0])
|
|
self.assertIs(first[1], second[1])
|
|
self.assertIsNone(block._modulation_cache)
|
|
self.assertEqual(block.img_mod[1].calls, 1)
|
|
self.assertEqual(block.txt_mod[1].calls, 1)
|
|
|
|
def test_tensor_identity_version_and_condition_invalidate_cache(self):
|
|
block = self._block()
|
|
timestep = torch.tensor([500.0], device="cuda")
|
|
hidden = torch.empty(1, 17, 32, device="cuda", dtype=torch.bfloat16)
|
|
temb = torch.randn(1, 32, device="cuda", dtype=torch.bfloat16)
|
|
key = self._key(timestep, hidden)
|
|
block._get_modulation_params(temb, temb, key)
|
|
|
|
with torch.no_grad():
|
|
timestep.add_(1)
|
|
mutated = self._key(timestep, hidden)
|
|
block._get_modulation_params(temb, temb, mutated)
|
|
self.assertEqual(block.img_mod[1].calls, 2)
|
|
|
|
same_value_new_tensor = self._key(timestep.clone(), hidden)
|
|
block._get_modulation_params(temb, temb, same_value_new_tensor)
|
|
self.assertEqual(block.img_mod[1].calls, 3)
|
|
|
|
condition = torch.tensor([1], device="cuda")
|
|
conditioned = self._key(timestep, hidden, condition)
|
|
block._get_modulation_params(temb, temb, conditioned)
|
|
self.assertEqual(block.img_mod[1].calls, 4)
|
|
|
|
def test_grad_enabled_path_disables_and_clears_cache(self):
|
|
block = self._block()
|
|
timestep = torch.tensor([500.0], device="cuda")
|
|
hidden = torch.empty(1, 17, 32, device="cuda", dtype=torch.bfloat16)
|
|
temb = torch.randn(1, 32, device="cuda", dtype=torch.bfloat16)
|
|
key = self._key(timestep, hidden)
|
|
block._get_modulation_params(temb, temb, key)
|
|
|
|
self.assertIsNone(_qwen_modulation_cache_key(timestep, None, hidden))
|
|
block._get_modulation_params(temb, temb, None)
|
|
|
|
self.assertIsNone(block._modulation_cache)
|
|
self.assertEqual(block.img_mod[1].calls, 2)
|
|
|
|
def test_inference_tensors_cache_and_graph_path_falls_back(self):
|
|
block = self._block()
|
|
with torch.inference_mode():
|
|
timestep = torch.tensor([500.0], device="cuda")
|
|
hidden = torch.empty(1, 17, 32, device="cuda", dtype=torch.bfloat16)
|
|
temb = torch.randn(1, 32, device="cuda", dtype=torch.bfloat16)
|
|
key = _qwen_modulation_cache_key(timestep, None, hidden)
|
|
|
|
first = block._get_modulation_params(temb, temb, key)
|
|
second = block._get_modulation_params(temb, temb, key)
|
|
|
|
self.assertIs(first[0], second[0])
|
|
self.assertEqual(block.img_mod[1].calls, 1)
|
|
|
|
with (
|
|
patch(
|
|
"sglang.multimodal_gen.runtime.models.dits.qwen_image.is_in_breakable_cuda_graph",
|
|
return_value=True,
|
|
),
|
|
torch.no_grad(),
|
|
):
|
|
self.assertIsNone(_qwen_modulation_cache_key(timestep, None, hidden))
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# GLM-Image -- LayerNorm + modulate and per-head qk LayerNorm
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize("shape", [(1, 4096, 4096), (2, 301, 4096), (1, 1, 2560)])
|
|
def test_glm_ln_modulate_is_bit_exact(shape):
|
|
# (1, 4096, 4096) is the real GLM-Image image-stream shape (1024^2,
|
|
# hidden 4096); the others cover the text stream and another hidden.
|
|
torch.manual_seed(0)
|
|
batch, seq, hidden = shape
|
|
norm = torch.nn.LayerNorm(hidden, eps=1e-5, elementwise_affine=False).cuda()
|
|
x = (torch.randn(batch, seq, hidden, device="cuda") * 8).bfloat16()
|
|
emb = torch.randn(batch, 12 * hidden, device="cuda").bfloat16()
|
|
chunks = emb.chunk(12, dim=1) # strided adaLN projection views
|
|
shift, scale = chunks[0], chunks[2]
|
|
out = _glm_ln_modulate(norm, x, scale, shift, x.dtype)
|
|
assert torch.equal(out, _glm_eager_ln_modulate(norm, x, scale, shift, x.dtype))
|
|
assert glm_image._GLM_LN_MOD.verified
|
|
assert not glm_image._GLM_LN_MOD.disabled
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize("shape", [(1, 4360, 32, 128), (2, 37, 3, 40), (1, 129, 5, 64)])
|
|
def test_glm_qk_head_layernorm_is_bit_exact(shape):
|
|
# (1, 4360, 32, 128) is the real GLM-Image q/k shape (text + image
|
|
# tokens, 32 heads of dim 128); the others cover partially-filled warps.
|
|
torch.manual_seed(1)
|
|
batch, seq, heads, head_dim = shape
|
|
norm_q = torch.nn.LayerNorm(head_dim, eps=1e-5, elementwise_affine=False).cuda()
|
|
norm_k = torch.nn.LayerNorm(head_dim, eps=1e-5, elementwise_affine=False).cuda()
|
|
q = (torch.randn(batch, seq, heads, head_dim, device="cuda") * 5).bfloat16()
|
|
k = (torch.randn(batch, seq, heads, head_dim, device="cuda") * 5).bfloat16()
|
|
q_out, k_out = _glm_qk_layernorm(norm_q, norm_k, q, k, q.dtype)
|
|
assert torch.equal(q_out, norm_q(q).to(q.dtype))
|
|
assert torch.equal(k_out, norm_k(k).to(k.dtype))
|
|
assert glm_image._GLM_QK_LN.verified
|
|
assert not glm_image._GLM_QK_LN.disabled
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Sana -- stream-conditional LayerNorm + modulate
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize(
|
|
"shape,nmod,transposed",
|
|
[
|
|
((2, 1024, 2240), 6, False),
|
|
((2, 1024, 2240), 2, False),
|
|
((1, 1024, 2240), 6, True),
|
|
((1, 37, 2240), 6, False),
|
|
],
|
|
)
|
|
def test_sana_fused_ln_modulate_is_bit_exact(shape, nmod, transposed):
|
|
# (., 1024, 2240) is the real Sana 1024px shape; hidden 2240 % 512 != 0
|
|
# exercises the kernel's partial tail chunk. nmod mirrors the two adaLN
|
|
# chunk layouts, transposed the permuted layout the Sana DiT serves.
|
|
torch.manual_seed(0)
|
|
batch, seq, hidden = shape
|
|
norm = torch.nn.LayerNorm(hidden, eps=1e-6, elementwise_affine=False).cuda()
|
|
x = (torch.randn(batch, seq, hidden, device="cuda") * 4).bfloat16()
|
|
if transposed:
|
|
x = x.permute(0, 2, 1).contiguous().permute(0, 2, 1)
|
|
emb = torch.randn(batch, nmod, hidden, device="cuda").bfloat16()
|
|
shift, scale = emb.chunk(nmod, dim=1)[0], emb.chunk(nmod, dim=1)[-1]
|
|
# default-stream eager serving must stay on the untouched eager chain
|
|
n_sigs = len(sana._SANA_LN_MOD.verified_sigs)
|
|
sana_ln_modulate(norm, x, scale, shift)
|
|
assert len(sana._SANA_LN_MOD.verified_sigs) == n_sigs
|
|
# The fusion engages on non-default streams (the BCG warmup/capture path).
|
|
# x/scale/shift were filled on the default stream, so the side stream must
|
|
# wait for that work before reading them -- without this the fused kernel
|
|
# can read a half-written tensor, the first-sight torch.equal check fails,
|
|
# and the gate disables itself *permanently*, which then breaks every later
|
|
# parametrization too. It only loses the race when the GPU is contended,
|
|
# which is why it shows up on shared CI runners and not on an idle box.
|
|
side = torch.cuda.Stream()
|
|
side.wait_stream(torch.cuda.current_stream())
|
|
with torch.cuda.stream(side):
|
|
out = sana_ln_modulate(norm, x, scale, shift)
|
|
assert len(sana._SANA_LN_MOD.verified_sigs) == n_sigs + 1 # verified
|
|
out2 = sana_ln_modulate(norm, x, scale, shift) # verified-sig lane
|
|
torch.cuda.current_stream().wait_stream(side)
|
|
torch.cuda.synchronize()
|
|
assert torch.equal(out, _sana_eager_ln_modulate(norm, x, scale, shift))
|
|
assert torch.equal(out2, out) and not sana._SANA_LN_MOD.disabled
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# ERNIE-Image -- bit-exact RMSNorm scale/shift and rotate-half RoPE
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize("shape", [(1, 4216, 4096), (2, 1140, 4096), (1, 128, 2048)])
|
|
def test_ernie_norm_scale_shift_is_bit_exact(shape):
|
|
# (1, 4216, 4096) is the real ERNIE-Image shape (1024^2 image + text
|
|
# tokens, hidden 4096); 2048 covers the threads_per_row=32 regime.
|
|
torch.manual_seed(0)
|
|
batch, seq, hidden = shape
|
|
norm = RMSNorm(hidden, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
with torch.no_grad():
|
|
norm.weight.copy_(torch.randn(hidden))
|
|
x = torch.randn(batch, seq, hidden, device="cuda", dtype=torch.bfloat16)
|
|
residual = torch.randn_like(x)
|
|
update = torch.randn_like(x)
|
|
scale = torch.randn(batch, 1, hidden, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
shift = torch.randn(batch, 1, hidden, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
gate = torch.randn(batch, 1, hidden, device="cuda", dtype=torch.bfloat16)
|
|
|
|
with torch.no_grad():
|
|
out = _ernie_norm_scale_shift(norm, x, scale, shift)
|
|
ref = norm(x) * (1 + scale) + shift
|
|
assert torch.equal(out, ref)
|
|
|
|
out2, res = _ernie_gated_norm_scale_shift(
|
|
norm, residual, update, gate, scale, shift
|
|
)
|
|
res_ref = residual + gate * update
|
|
ref2 = norm(res_ref) * (1 + scale) + shift
|
|
assert torch.equal(res, res_ref)
|
|
assert torch.equal(out2, ref2)
|
|
|
|
# the fast paths must actually be in use (not silently disabled)
|
|
assert ernie_image._ERNIE_NORM.verified
|
|
assert ernie_image._ERNIE_GATED_NORM.verified
|
|
assert not ernie_image._ERNIE_NORM.disabled
|
|
assert not ernie_image._ERNIE_GATED_NORM.disabled
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not is_cuda(), reason="the fused QKNorm+RoPE JIT kernel is CUDA-only"
|
|
)
|
|
def test_ernie_qknorm_rope_is_bit_exact():
|
|
torch.manual_seed(1)
|
|
ernie_image._ERNIE_QKNORM_ROPE.disabled = False
|
|
ernie_image._ERNIE_QKNORM_ROPE.verified = False
|
|
batch, seq, heads, head_dim = 1, 257, 32, 128
|
|
q = torch.randn(batch, seq, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
|
k = torch.randn_like(q)
|
|
q_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
k_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
cos = torch.randn(seq, head_dim, device="cuda", dtype=torch.bfloat16)
|
|
sin = torch.randn_like(cos)
|
|
cache = torch.cat((cos, sin), dim=-1).contiguous()
|
|
positions = torch.arange(seq, device="cuda", dtype=torch.long)
|
|
|
|
q_ref, k_ref = _ernie_qknorm_rope_reference(
|
|
q.clone(), k.clone(), q_norm, k_norm, head_dim, cos, sin
|
|
)
|
|
q_out, k_out = _ernie_qknorm_rope(
|
|
q,
|
|
k,
|
|
q_norm,
|
|
k_norm,
|
|
head_dim,
|
|
cos,
|
|
sin,
|
|
cache,
|
|
positions,
|
|
)
|
|
|
|
assert torch.equal(q_out, q_ref)
|
|
assert torch.equal(k_out, k_ref)
|
|
assert ernie_image._ERNIE_QKNORM_ROPE.verified
|
|
assert not ernie_image._ERNIE_QKNORM_ROPE.disabled
|
|
|
|
|
|
def test_ernie_qknorm_rope_first_attempt_exception_uses_pristine_inputs():
|
|
torch.manual_seed(2)
|
|
ernie_image._ERNIE_QKNORM_ROPE.disabled = False
|
|
ernie_image._ERNIE_QKNORM_ROPE.verified = False
|
|
batch, seq, heads, head_dim = 1, 17, 4, 128
|
|
q = torch.randn(batch, seq, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
|
k = torch.randn_like(q)
|
|
q_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
k_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
cos = torch.randn(seq, head_dim, device="cuda", dtype=torch.bfloat16)
|
|
sin = torch.randn_like(cos)
|
|
cache = torch.cat((cos, sin), dim=-1).contiguous()
|
|
positions = torch.arange(seq, device="cuda", dtype=torch.long)
|
|
q_ref, k_ref = _ernie_qknorm_rope_reference(
|
|
q.clone(), k.clone(), q_norm, k_norm, head_dim, cos, sin
|
|
)
|
|
|
|
def mutate_then_raise(**kwargs):
|
|
kwargs["q"].zero_()
|
|
kwargs["k"].zero_()
|
|
raise RuntimeError("synthetic kernel failure")
|
|
|
|
with patch.object(ernie_image, "apply_qk_norm_rope", mutate_then_raise):
|
|
q_out, k_out = _ernie_qknorm_rope(
|
|
q,
|
|
k,
|
|
q_norm,
|
|
k_norm,
|
|
head_dim,
|
|
cos,
|
|
sin,
|
|
cache,
|
|
positions,
|
|
)
|
|
|
|
assert torch.equal(q_out, q_ref)
|
|
assert torch.equal(k_out, k_ref)
|
|
assert ernie_image._ERNIE_QKNORM_ROPE.disabled
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# LongCat-Image -- full-width interleaved QKNorm + RoPE
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@requires_inline_ptx
|
|
def test_longcat_qknorm_rope_is_bit_exact():
|
|
torch.manual_seed(3)
|
|
batch, seq, heads, head_dim = 2, 17, 24, 128
|
|
offset = 11
|
|
q = torch.randn(batch, seq, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
|
k = torch.randn_like(q)
|
|
q_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
k_norm = RMSNorm(head_dim, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
with torch.no_grad():
|
|
q_norm.weight.copy_(torch.randn_like(q_norm.weight))
|
|
k_norm.weight.copy_(torch.randn_like(k_norm.weight))
|
|
|
|
cos = torch.randn(offset + seq, head_dim, device="cuda")
|
|
sin = torch.randn_like(cos)
|
|
image_rotary_emb = (cos[offset:], sin[offset:])
|
|
cache = torch.cat((cos, sin), dim=-1).contiguous()
|
|
positions = torch.arange(offset, offset + seq, device="cuda", dtype=torch.int64)
|
|
|
|
q_ref, k_ref = apply_qk_norm(q.clone(), k.clone(), q_norm, k_norm, head_dim)
|
|
q_ref = longcat_image.apply_rotary_emb(q_ref, image_rotary_emb, sequence_dim=1)
|
|
k_ref = longcat_image.apply_rotary_emb(k_ref, image_rotary_emb, sequence_dim=1)
|
|
|
|
q_fused, k_fused = q.clone(), k.clone()
|
|
q_out, k_out = _apply_longcat_qknorm_rope(
|
|
q_fused,
|
|
k_fused,
|
|
q_norm,
|
|
k_norm,
|
|
head_dim,
|
|
image_rotary_emb,
|
|
cache,
|
|
positions,
|
|
)
|
|
|
|
assert q_out.data_ptr() == q_fused.data_ptr()
|
|
assert k_out.data_ptr() == k_fused.data_ptr()
|
|
assert torch.equal(q_out, q_ref)
|
|
assert torch.equal(k_out, k_ref)
|
|
assert longcat_image._LONGCAT_QKNORM_ROPE.verified
|
|
assert not longcat_image._LONGCAT_QKNORM_ROPE.disabled
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# LTX-2 -- weightless RMSNorm + modulate (quality-gated)
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
def test_ltx2_qknorm_split_rope_hopper_site_is_quality_gated():
|
|
attention = nn.Module()
|
|
LTX2_QKNORM_SPLIT_ROPE.mark(attention)
|
|
assert not LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention)
|
|
assert LTX2_QKNORM_SPLIT_ROPE.mount(attention)
|
|
assert LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention)
|
|
LTX2_QKNORM_SPLIT_ROPE.unmount(attention)
|
|
assert not LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention)
|
|
|
|
|
|
def _ltx2_eager(rms, x, scale, shift, eps):
|
|
return rms(x, eps) * (1 + scale) + shift
|
|
|
|
|
|
def _ltx2_inputs(hidden, batch=1, seq=4096):
|
|
rms = RMSNormNoWeight()
|
|
x = torch.randn(batch, seq, hidden, device="cuda", dtype=torch.bfloat16)
|
|
scale = torch.randn(batch, 1, hidden, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
shift = torch.randn(batch, 1, hidden, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
return rms, x, scale, shift
|
|
|
|
|
|
# hidden 4096 = LTX-2 video stream, 2048 = audio stream.
|
|
@pytest.mark.parametrize("hidden", [4096, 2048])
|
|
def test_ltx2_lossless_default_is_bitexact(hidden):
|
|
# A marked-but-unmounted site uses only the self-verified bit-exact
|
|
# modulate fast path after the reference aten RMSNorm.
|
|
block = nn.Module()
|
|
LTX2_RMS_NORM_MODULATE.mark(block)
|
|
rms, x, scale, shift = _ltx2_inputs(hidden)
|
|
out = _ltx2_rms_norm_modulate(block, rms, x, scale, shift, 1e-6)
|
|
assert torch.equal(out, _ltx2_eager(rms, x, scale, shift, 1e-6))
|
|
|
|
|
|
def test_ltx2_lossless_compile_keeps_expression_visible_to_inductor(monkeypatch):
|
|
block = nn.Module()
|
|
LTX2_RMS_NORM_MODULATE.mark(block)
|
|
rms, x, scale, shift = _ltx2_inputs(2048, seq=126)
|
|
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
|
monkeypatch.setattr(
|
|
ltx2_module,
|
|
"_ltx2_modulate",
|
|
lambda *_args: pytest.fail("compiled path must not call the opaque custom op"),
|
|
)
|
|
|
|
out = _ltx2_rms_norm_modulate(block, rms, x, scale, shift, 1e-6)
|
|
assert torch.equal(out, _ltx2_eager(rms, x, scale, shift, 1e-6))
|
|
|
|
|
|
@requires_inline_ptx
|
|
@pytest.mark.parametrize("hidden", [4096, 2048])
|
|
def test_ltx2_mounted_high_uses_fused_kernel(hidden):
|
|
block = nn.Module()
|
|
LTX2_RMS_NORM_MODULATE.mark(block)
|
|
assert LTX2_RMS_NORM_MODULATE.mount(block)
|
|
try:
|
|
rms, x, scale, shift = _ltx2_inputs(hidden)
|
|
out = _ltx2_rms_norm_modulate(block, rms, x, scale, shift, 1e-6)
|
|
# The mounted path routes through the fused kernel exactly.
|
|
assert torch.equal(out, fused_ltx2_rms_norm_modulate(x, scale, shift, 1e-6))
|
|
# And stays within half-precision rounding of the eager reference.
|
|
ref = _ltx2_eager(rms, x, scale, shift, 1e-6)
|
|
assert torch.allclose(out.float(), ref.float(), atol=3e-2, rtol=1e-2)
|
|
finally:
|
|
LTX2_RMS_NORM_MODULATE.unmount(block)
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# HunyuanVideo -- QKV/RoPE pack and quality-gated QK RMSNorm
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("img_tokens,txt_tokens", [(257, 31), (4096, 256)])
|
|
def test_hunyuan_qkv_rope_pack_is_bit_exact(img_tokens, txt_tokens):
|
|
torch.manual_seed(0)
|
|
shape_img = (1, img_tokens, 24, 128)
|
|
shape_txt = (1, txt_tokens, 24, 128)
|
|
img_q, img_k, img_v = (
|
|
torch.randn(shape_img, device="cuda", dtype=torch.bfloat16) for _ in range(3)
|
|
)
|
|
txt_q, txt_k, txt_v = (
|
|
torch.randn(shape_txt, device="cuda", dtype=torch.bfloat16) for _ in range(3)
|
|
)
|
|
cos = torch.randn(img_tokens, 64, device="cuda")
|
|
sin = torch.randn_like(cos)
|
|
|
|
q, k, v = _hunyuan_pack_qkv(img_q, img_k, img_v, txt_q, txt_k, txt_v, cos, sin)
|
|
q_ref = torch.cat(
|
|
(_apply_rotary_emb(img_q, cos, sin, is_neox_style=False), txt_q), dim=1
|
|
)
|
|
k_ref = torch.cat(
|
|
(_apply_rotary_emb(img_k, cos, sin, is_neox_style=False), txt_k), dim=1
|
|
)
|
|
v_ref = torch.cat((img_v, txt_v), dim=1)
|
|
|
|
assert torch.equal(q, q_ref)
|
|
assert torch.equal(k, k_ref)
|
|
assert torch.equal(v, v_ref)
|
|
|
|
|
|
def test_hunyuan_qkv_rope_pack_uses_int64_row_offsets():
|
|
if torch.cuda.get_device_properties(0).total_memory < 16 * 2**30:
|
|
pytest.skip("needs >= 16 GB GPU memory")
|
|
|
|
img_tokens, txt_tokens = 115200, 8
|
|
num_heads, head_dim = 24, 128
|
|
total_tokens = img_tokens + txt_tokens
|
|
projection_width = 21504
|
|
|
|
projection = torch.zeros(
|
|
(1, total_tokens, projection_width),
|
|
device="cuda",
|
|
dtype=torch.bfloat16,
|
|
)
|
|
qkv = projection[..., : 3 * num_heads * head_dim].view(
|
|
1, total_tokens, 3, num_heads, head_dim
|
|
)
|
|
q = qkv[:, :, 0].contiguous()
|
|
k = qkv[:, :, 1].contiguous()
|
|
v = qkv[:, :, 2]
|
|
assert (img_tokens - 1) * v.stride(1) > torch.iinfo(torch.int32).max
|
|
|
|
cos = torch.ones((img_tokens, head_dim // 2), device="cuda")
|
|
sin = torch.zeros_like(cos)
|
|
packed = hunyuan_qkv_rope_pack(
|
|
q[:, :img_tokens],
|
|
k[:, :img_tokens],
|
|
v[:, :img_tokens],
|
|
q[:, img_tokens:],
|
|
k[:, img_tokens:],
|
|
v[:, img_tokens:],
|
|
cos,
|
|
sin,
|
|
)
|
|
torch.cuda.synchronize()
|
|
|
|
expected_shape = (1, total_tokens, num_heads, head_dim)
|
|
assert all(x.shape == expected_shape for x in packed)
|
|
assert all(x[0, img_tokens - 1, -1, -1].item() == 0 for x in packed)
|
|
|
|
|
|
@pytest.mark.skipif(not is_cuda(), reason="FlashInfer CuTe QK RMSNorm is CUDA-only")
|
|
def test_hunyuan_quality_qknorm_matches_rmsnorm():
|
|
torch.manual_seed(1)
|
|
site = torch.nn.Module()
|
|
HUNYUAN_QKNORM.mark(site)
|
|
q_norm = RMSNorm(128, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
k_norm = RMSNorm(128, eps=1e-6).to(device="cuda", dtype=torch.bfloat16)
|
|
packed = torch.randn(1, 257, 3, 24, 128, device="cuda", dtype=torch.bfloat16)
|
|
q, k = packed[:, :, 0], packed[:, :, 1]
|
|
q_ref = q_norm(q.contiguous()).to(q)
|
|
k_ref = k_norm(k.contiguous()).to(k)
|
|
|
|
q_unmounted, k_unmounted = _hunyuan_qknorm(site, q, k, q_norm, k_norm)
|
|
assert torch.equal(q_unmounted, q_ref)
|
|
assert torch.equal(k_unmounted, k_ref)
|
|
|
|
assert HUNYUAN_QKNORM.mount(site)
|
|
q_out, k_out = _hunyuan_qknorm(site, q, k, q_norm, k_norm)
|
|
torch.testing.assert_close(q_out, q_ref, atol=2e-2, rtol=2e-2)
|
|
torch.testing.assert_close(k_out, k_ref, atol=2e-2, rtol=2e-2)
|
|
|
|
HUNYUAN_QKNORM.unmount(site)
|
|
q_unmounted, k_unmounted = _hunyuan_qknorm(site, q, k, q_norm, k_norm)
|
|
assert torch.equal(q_unmounted, q_ref)
|
|
assert torch.equal(k_unmounted, k_ref)
|
|
|
|
|
|
def test_hunyuan_quality_qknorm_stays_unmounted_without_cute_kernel():
|
|
site = torch.nn.Module()
|
|
HUNYUAN_QKNORM.mark(site)
|
|
|
|
with patch.object(hunyuan_qknorm, "_get_qk_rmsnorm_cute", return_value=None):
|
|
assert not HUNYUAN_QKNORM.mount(site)
|
|
|
|
assert not HUNYUAN_QKNORM.is_enabled(site)
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Wan VAE -- fused RMSNorm+SiLU module gate
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
def _wan_cl3d(shape, dtype):
|
|
return torch.randn(shape, device="cuda", dtype=dtype).contiguous(
|
|
memory_format=torch.channels_last_3d
|
|
)
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.parametrize(
|
|
"norm_cls,frames",
|
|
[(WanRMS_norm, 3), (qwen_vae.QwenImageRMS_norm, 1)],
|
|
ids=["wan", "qwen"],
|
|
)
|
|
def test_wan_and_qwen_vae_norm_gate_dispatch(norm_cls, frames) -> None:
|
|
# Gate off must stay bit-exact; gate on must route to the fused kernel.
|
|
torch.cuda.manual_seed(0)
|
|
norm = norm_cls(96, images=False).to(device="cuda", dtype=torch.bfloat16)
|
|
norm.gamma.add_(torch.randn_like(norm.gamma))
|
|
gate = VaeFastPathGate()
|
|
fused = FusedWanRMSNormSiLU(norm, gate)
|
|
# Parameter names must not change (weight transfer matches by name).
|
|
assert [n for n, _ in fused.named_parameters()] == ["gamma"]
|
|
x = _wan_cl3d((1, 96, frames, 10, 14), torch.bfloat16)
|
|
assert torch.equal(fused(x), nn.SiLU()(norm(x)))
|
|
gate.enabled = True
|
|
expected = wan_rmsnorm_silu(x, norm.gamma, rms_scale=float(norm.scale))
|
|
assert torch.equal(fused(x), expected)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_wan_vae_rejects_empty_input() -> None:
|
|
x = torch.empty(1, 96, 0, 2, 2, device="cuda", dtype=torch.bfloat16).to(
|
|
memory_format=torch.channels_last_3d
|
|
)
|
|
gamma = torch.ones(96, 1, 1, 1, device="cuda", dtype=torch.bfloat16)
|
|
assert not can_use_wan_rmsnorm_silu(x, gamma, None)
|
|
|
|
|
|
def _wan_tiny_decoder(seed=0):
|
|
torch.manual_seed(seed)
|
|
dec = wanvae.WanDecoder3d(
|
|
dim=16, z_dim=4, dim_mult=[1, 1], num_res_blocks=1, temperal_upsample=[True]
|
|
).to("cuda", torch.bfloat16)
|
|
for m in dec.modules():
|
|
if isinstance(m, nn.Conv3d):
|
|
m.weight.data = m.weight.data.to(memory_format=torch.channels_last_3d)
|
|
return dec
|
|
|
|
|
|
def _wan_decode_chunks(dec, chunks):
|
|
# Mirrors AutoencoderKLWan.decode: one latent frame per chunk, feature
|
|
# cache threaded through the contextvars the decoder reads.
|
|
n_conv = sum(isinstance(m, wanvae.WanCausalConv3d) for m in dec.modules())
|
|
cache = [None] * n_conv
|
|
outs = []
|
|
for i, z in enumerate(chunks):
|
|
tok_c = wanvae.feat_cache.set(cache)
|
|
tok_i = wanvae.feat_idx.set(0)
|
|
tok_f = wanvae.first_chunk.set(i == 0)
|
|
try:
|
|
outs.append(dec(z))
|
|
finally:
|
|
wanvae.feat_cache.reset(tok_c)
|
|
wanvae.feat_idx.reset(tok_i)
|
|
wanvae.first_chunk.reset(tok_f)
|
|
return torch.cat(outs, dim=2)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_wan_vae_conv_epilogue_wiring_is_bit_exact() -> None:
|
|
# The lossless decoder (fused cat+pad, conv bias epilogue, bias+residual
|
|
# fold, bias deferred into norm2) must equal the pure aten module chain.
|
|
dec = _wan_tiny_decoder()
|
|
chunks = [_wan_cl3d((1, 4, 1, 6, 6), torch.bfloat16) for _ in range(3)]
|
|
fast = _wan_decode_chunks(dec, chunks)
|
|
saved = (
|
|
wanvae.cat_pad_channels_last_3d,
|
|
wanvae.conv_bias_epilogue,
|
|
wanvae.can_use_conv_bias_epilogue,
|
|
)
|
|
wanvae.cat_pad_channels_last_3d = None
|
|
wanvae.conv_bias_epilogue = None
|
|
wanvae.can_use_conv_bias_epilogue = None
|
|
try:
|
|
eager = _wan_decode_chunks(dec, chunks)
|
|
finally:
|
|
(
|
|
wanvae.cat_pad_channels_last_3d,
|
|
wanvae.conv_bias_epilogue,
|
|
wanvae.can_use_conv_bias_epilogue,
|
|
) = saved
|
|
assert torch.equal(fast, eager)
|
|
# Deferred conv1 bias through the gated wrapper, gate off: still exact.
|
|
gate = VaeFastPathGate()
|
|
wan_vae_cuda_opt._install_norm_silu(
|
|
dec,
|
|
gate,
|
|
residual_block_cls=wanvae.WanResidualBlock,
|
|
rms_norm_cls=WanRMS_norm,
|
|
label="test",
|
|
)
|
|
assert torch.equal(_wan_decode_chunks(dec, chunks), eager)
|
|
gate.enabled = True
|
|
out = _wan_decode_chunks(dec, chunks)
|
|
rel = ((out.float() - eager.float()).norm() / eager.float().norm()).item()
|
|
assert rel < 2e-2, rel
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_wan_vae_time_interleave_matches_stack_and_keeps_layout() -> None:
|
|
# time_conv output [B, 2C, T, H, W] -> interleaved [B, C, 2T, H, W].
|
|
b, c, t, h, w = 1, 8, 3, 6, 10
|
|
x = _wan_cl3d((b, 2 * c, t, h, w), torch.bfloat16)
|
|
ref = torch.stack(
|
|
(x.reshape(b, 2, c, t, h, w)[:, 0], x.reshape(b, 2, c, t, h, w)[:, 1]), 3
|
|
).reshape(b, c, 2 * t, h, w)
|
|
|
|
class _Holder: # stands in for the WanResample instance
|
|
pass
|
|
|
|
holder = _Holder()
|
|
off = wanvae._interleave_time_pairs(holder, x, b, c, t, h, w)
|
|
assert torch.equal(off, ref) and off.is_contiguous() # eager NCDHW path
|
|
holder._sgl_gate = VaeFastPathGate()
|
|
holder._sgl_gate.enabled = True
|
|
on = wanvae._interleave_time_pairs(holder, x, b, c, t, h, w)
|
|
assert torch.equal(on, ref)
|
|
assert on.is_contiguous(memory_format=torch.channels_last_3d)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_wan_vae_upsample_wrapper_dispatch() -> None:
|
|
gate = VaeFastPathGate()
|
|
up = GatedChannelsLastUpsample(
|
|
wanvae.WanUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), gate
|
|
)
|
|
aten = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact")
|
|
# Canonical NHWC (multi-frame chunk): the Triton gather replaces aten's
|
|
# NHWC kernel on both paths, same values and layout.
|
|
x = torch.randn(4, 8, 6, 6, device="cuda", dtype=torch.bfloat16).contiguous(
|
|
memory_format=torch.channels_last
|
|
)
|
|
out = up(x)
|
|
assert torch.equal(out, aten(x))
|
|
assert out.is_contiguous(memory_format=torch.channels_last)
|
|
# Degenerate batch stride (single frame): gate off keeps aten's NCHW
|
|
# result, gate on canonicalises and stays channels_last.
|
|
x1 = (
|
|
_wan_cl3d((1, 8, 1, 6, 6), torch.bfloat16)
|
|
.permute(0, 2, 1, 3, 4)
|
|
.reshape(1, 8, 6, 6)
|
|
)
|
|
ref1 = aten(x1)
|
|
off = up(x1)
|
|
assert torch.equal(off, ref1) and off.is_contiguous()
|
|
gate.enabled = True
|
|
on = up(x1)
|
|
assert torch.equal(on, ref1)
|
|
assert on.is_contiguous(memory_format=torch.channels_last)
|
|
# NCHW input is untouched on either path.
|
|
xn = torch.randn(2, 8, 6, 6, device="cuda", dtype=torch.bfloat16)
|
|
assert torch.equal(up(xn), aten(xn)) and up(xn).is_contiguous()
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_wan_vae_decoder_install_wires_resample_gate() -> None:
|
|
torch.manual_seed(0)
|
|
dec = wanvae.WanDecoder3d(
|
|
dim=16, z_dim=4, dim_mult=[1, 1], num_res_blocks=1, temperal_upsample=[True]
|
|
).to("cuda", torch.bfloat16)
|
|
keys = set(dec.state_dict())
|
|
gate = VaeFastPathGate()
|
|
n_norm = wan_vae_cuda_opt._install_norm_silu(
|
|
dec,
|
|
gate,
|
|
residual_block_cls=wanvae.WanResidualBlock,
|
|
rms_norm_cls=WanRMS_norm,
|
|
label="test",
|
|
)
|
|
n_up = wan_vae_cuda_opt._install_channels_last_upsample(
|
|
dec, gate, wanvae.WanUpsample
|
|
)
|
|
n_gated = wan_vae_cuda_opt._install_module_gates(
|
|
dec, gate, (wanvae.WanResample, wanvae.WanAttentionBlock)
|
|
)
|
|
# one Resample plus the mid block's attention
|
|
assert n_norm == 13 and n_up == 1 and n_gated == 2
|
|
assert set(dec.state_dict()) == keys
|
|
resample = next(m for m in dec.modules() if type(m) is wanvae.WanResample)
|
|
assert resample._sgl_gate is gate
|
|
attn = next(m for m in dec.modules() if type(m) is wanvae.WanAttentionBlock)
|
|
assert attn._sgl_gate is gate
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Qwen-Image VAE (Wan 2.1 VAE) -- lossless causal-conv data movement and the
|
|
# quality-gated RMSNorm+SiLU / channels_last upsample fast path
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
def _qwen_causal_conv(cin, cout, channels_last=True):
|
|
conv = qwen_vae.QwenImageCausalConv3d(cin, cout, 3, padding=1).to(
|
|
"cuda", torch.bfloat16
|
|
)
|
|
if channels_last:
|
|
conv.weight.data = conv.weight.data.to(memory_format=torch.channels_last_3d)
|
|
return conv
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="the fused cat+pad kernel is CUDA-only")
|
|
def test_qwen_vae_causal_conv_cat_pad_is_bit_exact() -> None:
|
|
# Fused cat + pad + relayout must reproduce the aten chain for no cache,
|
|
# a one-frame cache (partial zero fill) and a full two-frame cache.
|
|
torch.manual_seed(0)
|
|
conv = _qwen_causal_conv(8, 16)
|
|
ref = _qwen_causal_conv(8, 16, channels_last=False)
|
|
ref.load_state_dict(conv.state_dict())
|
|
x = _wan_cl3d((1, 8, 1, 12, 10), torch.bfloat16)
|
|
for cache_t in (0, 1, 2):
|
|
cache = _wan_cl3d((1, 8, cache_t, 12, 10), torch.bfloat16) if cache_t else None
|
|
# The reference is the eager chain on the same (channels_last) weights.
|
|
expected = nn.Conv3d.forward(
|
|
conv,
|
|
qwen_vae.causal_conv3d_cat_pad(x, cache, list(conv._padding)).contiguous(
|
|
memory_format=torch.channels_last_3d
|
|
),
|
|
)
|
|
assert qwen_vae._fused_conv_cache_supported(conv, x)
|
|
assert torch.equal(conv(x, cache), expected)
|
|
# Contiguous weights take the original path unchanged.
|
|
assert not qwen_vae._fused_conv_cache_supported(ref, x)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen_vae_run_cached_causal_conv_matches_eager_bookkeeping() -> None:
|
|
# Stream three single-frame chunks (the image decode shape, and the first
|
|
# chunks of a clip) through one conv slot on the fused and the eager path;
|
|
# outputs and cache contents must agree, including the "Rep" slot.
|
|
torch.manual_seed(0)
|
|
fused = _qwen_causal_conv(8, 8)
|
|
eager = _qwen_causal_conv(8, 8, channels_last=False)
|
|
eager.load_state_dict(fused.state_dict())
|
|
chunks = [_wan_cl3d((1, 8, 1, 6, 6), torch.bfloat16) for _ in range(3)]
|
|
for first in (None, "Rep"):
|
|
fused_cache, eager_cache = [first], [first]
|
|
for x in chunks:
|
|
out_f = qwen_vae._run_cached_causal_conv(fused, x, fused_cache, 0)
|
|
out_e = qwen_vae._run_cached_causal_conv(eager, x, eager_cache, 0)
|
|
assert torch.equal(out_f, out_e)
|
|
# The fused path only ever stores tensors that the eager path would
|
|
# also consume: same values, possibly wider (never narrower).
|
|
fc, ec = fused_cache[0], eager_cache[0]
|
|
assert fc.shape[2] >= ec.shape[2]
|
|
assert torch.equal(fc[:, :, -ec.shape[2] :], ec)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen_vae_upsample_skips_fp32_round_trip_bit_exactly() -> None:
|
|
up = qwen_vae.QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact")
|
|
x = torch.randn(1, 8, 12, 10, device="cuda", dtype=torch.bfloat16).contiguous(
|
|
memory_format=torch.channels_last
|
|
)
|
|
assert torch.equal(up(x), nn.Upsample.forward(up, x.float()).type_as(x))
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen_vae_channels_last_upsample_keeps_layout() -> None:
|
|
gate = VaeFastPathGate()
|
|
up = GatedChannelsLastUpsample(
|
|
qwen_vae.QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"),
|
|
gate,
|
|
)
|
|
x5 = _wan_cl3d((1, 8, 1, 6, 6), torch.bfloat16)
|
|
x = x5.permute(0, 2, 1, 3, 4).reshape(1, 8, 6, 6) # degenerate batch stride
|
|
ref = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact")(x)
|
|
off = up(x)
|
|
assert torch.equal(off, ref) and off.is_contiguous() # NCHW, as today
|
|
gate.enabled = True
|
|
on = up(x)
|
|
assert torch.equal(on, ref)
|
|
assert on.is_contiguous(memory_format=torch.channels_last)
|
|
assert on.stride() == (12 * 12 * 8, 1, 12 * 8, 8)
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen_vae_decoder_install_is_gated_and_close() -> None:
|
|
torch.manual_seed(0)
|
|
dec = qwen_vae.QwenImageDecoder3d(
|
|
dim=16, z_dim=4, dim_mult=[1, 1], num_res_blocks=1, temperal_upsample=[False]
|
|
).to("cuda", torch.bfloat16)
|
|
for m in dec.modules():
|
|
if isinstance(m, nn.Conv3d):
|
|
m.weight.data = m.weight.data.to(memory_format=torch.channels_last_3d)
|
|
for p in dec.parameters():
|
|
p.add_(0.1 * torch.randn_like(p))
|
|
keys = set(dec.state_dict())
|
|
z = _wan_cl3d((1, 4, 1, 6, 6), torch.bfloat16)
|
|
ref = dec(z, feat_cache=[None] * 64, feat_idx=[0])
|
|
|
|
gate = VaeFastPathGate()
|
|
n_norm = wan_vae_cuda_opt._install_norm_silu(
|
|
dec,
|
|
gate,
|
|
residual_block_cls=qwen_vae.QwenImageResidualBlock,
|
|
rms_norm_cls=qwen_vae.QwenImageRMS_norm,
|
|
label="test",
|
|
)
|
|
n_up = wan_vae_cuda_opt._install_channels_last_upsample(
|
|
dec, gate, qwen_vae.QwenImageUpsample
|
|
)
|
|
# mid block (2 resnets) + 2 up blocks x (num_res_blocks + 1) resnets
|
|
# -> 6 resnets x 2 norms + norm_out; only the first up block upsamples.
|
|
assert n_norm == 13 and n_up == 1
|
|
assert set(dec.state_dict()) == keys
|
|
assert torch.equal(dec(z, feat_cache=[None] * 64, feat_idx=[0]), ref)
|
|
gate.enabled = True
|
|
out = dec(z, feat_cache=[None] * 64, feat_idx=[0])
|
|
assert out.shape == ref.shape
|
|
# Not bit-exact (fp32 statistics in the fused norm), but a bf16-rounding
|
|
# level perturbation: bound the relative error energy over the whole
|
|
# output rather than per element (random weights, unnormalised scale).
|
|
rel = ((out.float() - ref.float()).norm() / ref.float().norm()).item()
|
|
assert rel < 2e-2, rel
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# FLUX.2 VAE -- fused GroupNorm+SiLU and folded 2x upsample conv
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_flux2_vae_fast_path():
|
|
torch.manual_seed(0)
|
|
gate = vae_opt.VaeFastPathGate()
|
|
gn = nn.GroupNorm(32, 128, eps=1e-6).to("cuda", torch.bfloat16)
|
|
x = torch.randn(1, 128, 64, 64, device="cuda", dtype=torch.bfloat16).to(
|
|
memory_format=torch.channels_last
|
|
)
|
|
ref = F.silu(gn(x))
|
|
fused_gn = vae_opt.FusedGroupNormSiLU(gn, gate)
|
|
assert set(fused_gn.state_dict()) == {"weight", "bias"}
|
|
assert torch.equal(fused_gn(x), ref) # gate off: bit-exact reference
|
|
|
|
gate.enabled = True
|
|
fast = fused_gn(x)
|
|
assert fast.is_contiguous(memory_format=torch.channels_last)
|
|
torch.testing.assert_close(fast.float(), ref.float(), atol=0.06, rtol=0)
|
|
|
|
gate.enabled = False
|
|
up = Upsample2D(channels=32, use_conv=True).to("cuda", torch.bfloat16)
|
|
fused_up = vae_opt.FusedUpsample2xConv2d(up, gate)
|
|
assert set(fused_up.state_dict()) == {"conv.weight", "conv.bias"}
|
|
x = torch.randn(2, 32, 33, 29, device="cuda", dtype=torch.bfloat16)
|
|
ref = up(x)
|
|
assert torch.equal(fused_up(x), ref)
|
|
assert fused_up._fused_weight is None
|
|
|
|
gate.enabled = True
|
|
fast = fused_up(x)
|
|
assert fused_up._fused_weight is not None
|
|
ref_range = ref.float().max() - ref.float().min()
|
|
relative_mse = F.mse_loss(fast.float(), ref.float()) / ref_range.square()
|
|
assert relative_mse < 3.2e-5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AutoencoderKL (generic) -- fast-path install must not disturb the checkpoint
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _small_config():
|
|
config = StableDiffusion3VAEConfig()
|
|
config.arch_config.latent_channels = 2
|
|
config.arch_config.block_out_channels = (4, 4)
|
|
config.arch_config.down_block_types = ("DownEncoderBlock2D",) * 2
|
|
config.arch_config.up_block_types = ("UpDecoderBlock2D",) * 2
|
|
config.arch_config.layers_per_block = 1
|
|
config.arch_config.norm_num_groups = 1
|
|
config.arch_config.sample_size = 8
|
|
return config
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_autoencoder_kl_fastpath_install():
|
|
torch.manual_seed(0)
|
|
vae = AutoencoderKL(_small_config()).to("cuda", torch.bfloat16).eval()
|
|
ref_names = {n for n, _ in vae.named_parameters()}
|
|
ref_sd = {k: v.clone() for k, v in vae.state_dict().items()}
|
|
z = torch.randn(1, 2, 8, 8, device="cuda", dtype=torch.bfloat16)
|
|
ref = vae.decode(z)
|
|
|
|
opt = vae_opt.maybe_optimize_autoencoder_kl(vae)
|
|
# Wrappers must not change parameter FQNs; strict load must round-trip.
|
|
assert {n for n, _ in opt.named_parameters()} == ref_names
|
|
opt.load_state_dict(ref_sd, strict=True)
|
|
# Gate off: bit-for-bit the original path.
|
|
assert torch.equal(opt.decode(z), ref)
|
|
# use_vae_fast_path() is a no-op when nothing registered a gate, so check
|
|
# the wrappers went in before relying on it to switch paths.
|
|
assert any(
|
|
isinstance(m, (vae_opt.FusedGroupNormSiLU, vae_opt.FusedUpsample2xConv2d))
|
|
for m in opt.modules()
|
|
)
|
|
# Gate on: fast path runs and stays close; leaving the scope restores exact.
|
|
with use_vae_fast_path(opt, True):
|
|
torch.testing.assert_close(opt.decode(z).float(), ref.float(), atol=0.1, rtol=0)
|
|
assert torch.equal(opt.decode(z), ref)
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="the fused Q/K norm is CUDA-only")
|
|
def test_qwen21_qk_norm_verifies_and_preserves_native_fallback(monkeypatch):
|
|
x = torch.randn(1, 257, 8, 128, device="cuda", dtype=torch.bfloat16)
|
|
norm = qwen_image21.RMSNorm(
|
|
128, 1e-6, cast_x_before_out_mul=True, force_native=True
|
|
).to(device=x.device, dtype=x.dtype)
|
|
norm.weight.normal_()
|
|
expected = norm(x)
|
|
gate = BitExactFusionGate("test Q/K norm")
|
|
monkeypatch.setattr(qwen_image21, "_QK_NORM_FUSION", gate)
|
|
assert torch.equal(qwen_image21.apply_qk_norm(x, norm), expected)
|
|
assert gate.verified and not gate.disabled
|
|
x.normal_()
|
|
assert torch.equal(qwen_image21.apply_qk_norm(x, norm), norm(x))
|
|
|
|
gate = BitExactFusionGate("test mismatched Q/K norm")
|
|
monkeypatch.setattr(qwen_image21, "_QK_NORM_FUSION", gate)
|
|
monkeypatch.setattr(
|
|
qwen_image21,
|
|
"rmsnorm_preserve_reduction",
|
|
lambda x, weight, eps: torch.zeros_like(x),
|
|
)
|
|
assert torch.equal(qwen_image21.apply_qk_norm(x, norm), norm(x))
|
|
assert gate.disabled and not gate.verified
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen21_qk_norm_does_not_verify_during_capture(monkeypatch):
|
|
x = torch.randn(1, 17, 2, 128, device="cuda", dtype=torch.bfloat16)
|
|
norm = qwen_image21.RMSNorm(
|
|
128, 1e-6, cast_x_before_out_mul=True, force_native=True
|
|
).to(device=x.device, dtype=x.dtype)
|
|
norm(x)
|
|
gate = BitExactFusionGate("test captured Q/K norm")
|
|
monkeypatch.setattr(qwen_image21, "_QK_NORM_FUSION", gate)
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
out = qwen_image21.apply_qk_norm(x, norm)
|
|
assert not gate.verified and not gate.disabled
|
|
x.normal_()
|
|
graph.replay()
|
|
assert torch.equal(out, norm(x))
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="the fused LayerNorm modulation is CUDA-only")
|
|
def test_qwen21_modulation_verifies_and_preserves_native_fallback(monkeypatch):
|
|
x = torch.randn(1, 257, 4096, device="cuda", dtype=torch.bfloat16)
|
|
scale = torch.randn(1, 1, 4096, device=x.device, dtype=x.dtype)
|
|
norm = torch.nn.LayerNorm(4096, eps=1e-6, elementwise_affine=False).cuda()
|
|
gate = BitExactFusionGate("test scale-only modulation")
|
|
monkeypatch.setattr(qwen_image21, "_MODULATION_FUSION", gate)
|
|
expected = norm(x) * (1 + scale)
|
|
actual = qwen_image21.apply_modulation(x, norm, scale)
|
|
assert torch.equal(actual.view(torch.int16), expected.view(torch.int16))
|
|
assert gate.verified and not gate.disabled
|
|
x.normal_()
|
|
scale.normal_()
|
|
assert torch.equal(
|
|
qwen_image21.apply_modulation(x, norm, scale), norm(x) * (1 + scale)
|
|
)
|
|
|
|
gate = BitExactFusionGate("test mismatched modulation")
|
|
monkeypatch.setattr(qwen_image21, "_MODULATION_FUSION", gate)
|
|
monkeypatch.setattr(
|
|
qwen_image21,
|
|
"fused_layernorm_modulate",
|
|
lambda x, scale, shift, eps: torch.zeros_like(x),
|
|
)
|
|
assert torch.equal(
|
|
qwen_image21.apply_modulation(x, norm, scale), norm(x) * (1 + scale)
|
|
)
|
|
assert gate.disabled and not gate.verified
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_qwen21_modulation_does_not_verify_during_capture(monkeypatch):
|
|
x = torch.randn(1, 17, 128, device="cuda", dtype=torch.bfloat16)
|
|
scale = torch.randn(1, 1, 128, device=x.device, dtype=x.dtype)
|
|
norm = torch.nn.LayerNorm(128, eps=1e-6, elementwise_affine=False).cuda()
|
|
norm(x)
|
|
gate = BitExactFusionGate("test captured modulation")
|
|
monkeypatch.setattr(qwen_image21, "_MODULATION_FUSION", gate)
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
out = qwen_image21.apply_modulation(x, norm, scale)
|
|
assert not gate.verified and not gate.disabled
|
|
x.normal_()
|
|
scale.normal_()
|
|
graph.replay()
|
|
assert torch.equal(out, norm(x) * (1 + scale))
|
|
|
|
|
|
requires_nvidia_jit = pytest.mark.skipif(
|
|
not is_cuda(), reason="the Qwen-Image 2.1 JIT CUDA kernels are NVIDIA-only"
|
|
)
|
|
|
|
|
|
def _qwen21_rope(tokens: int, device: torch.device) -> torch.Tensor:
|
|
angles = torch.randn(tokens, 64, device=device, dtype=torch.float32)
|
|
return torch.polar(torch.ones_like(angles), angles)
|
|
|
|
|
|
@requires_nvidia_jit
|
|
@torch.no_grad()
|
|
def test_qwen21_cuda_qk_rope_verifies_and_preserves_native_fallback(monkeypatch):
|
|
x = torch.randn(1, 257, 8, 128, device="cuda", dtype=torch.bfloat16)
|
|
rope = _qwen21_rope(257, x.device)
|
|
norm = qwen_image21.RMSNorm(
|
|
128, 1e-6, cast_x_before_out_mul=True, force_native=True
|
|
).to(device=x.device, dtype=x.dtype)
|
|
norm.weight.normal_()
|
|
expected = qwen_image21.apply_rope(norm(x), rope)
|
|
gate = BitExactFusionGate("test CUDA Q/K norm + RoPE")
|
|
monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate)
|
|
assert torch.equal(qwen_image21.apply_qk_norm_rope(x, norm, rope), expected)
|
|
assert gate.verified and not gate.disabled
|
|
x.normal_()
|
|
assert torch.equal(
|
|
qwen_image21.apply_qk_norm_rope(x, norm, rope),
|
|
qwen_image21.apply_rope(norm(x), rope),
|
|
)
|
|
|
|
gate = BitExactFusionGate("test mismatched CUDA Q/K norm + RoPE")
|
|
monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate)
|
|
monkeypatch.setattr(
|
|
qwen_image21,
|
|
"qknorm_complex_rope_cuda",
|
|
lambda x, weight, rope, eps: torch.zeros_like(x),
|
|
)
|
|
assert torch.equal(
|
|
qwen_image21.apply_qk_norm_rope(x, norm, rope),
|
|
qwen_image21.apply_rope(norm(x), rope),
|
|
)
|
|
assert gate.disabled and not gate.verified
|
|
|
|
|
|
@requires_nvidia_jit
|
|
@torch.no_grad()
|
|
def test_qwen21_cuda_qk_rope_does_not_verify_during_capture(monkeypatch):
|
|
x = torch.randn(1, 17, 2, 128, device="cuda", dtype=torch.bfloat16)
|
|
rope = _qwen21_rope(17, x.device)
|
|
norm = qwen_image21.RMSNorm(
|
|
128, 1e-6, cast_x_before_out_mul=True, force_native=True
|
|
).to(device=x.device, dtype=x.dtype)
|
|
qwen_image21.apply_rope(norm(x), rope)
|
|
gate = BitExactFusionGate("test captured CUDA Q/K norm + RoPE")
|
|
monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate)
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
out = qwen_image21.apply_qk_norm_rope(x, norm, rope)
|
|
assert not gate.verified and not gate.disabled
|
|
x.normal_()
|
|
graph.replay()
|
|
assert torch.equal(out, qwen_image21.apply_rope(norm(x), rope))
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Qwen-Image 2.1 VAE -- bit-exact norm / bias fusions and the quality-gated channels_last decode
|
|
# -------------------------------------------------------------------------
|
|
|
|
|
|
def _qwen21_vae():
|
|
# The real latent channel count and 1/9 of the decoder width: every fold
|
|
# site has K >= 576 like the real decoder, and at a 64x64 latent all of them
|
|
# produce >= 2**18 output elements, so the first-sight comparisons decide.
|
|
ac = QwenImage21VAEArchConfig(
|
|
base_dim=16,
|
|
decoder_base_dim=16,
|
|
z_dim=64,
|
|
dim_mult=(1, 2, 4, 4, 4),
|
|
num_res_blocks=1,
|
|
temperal_downsample=(False, False, False, False),
|
|
in_channels=4,
|
|
out_channels=4,
|
|
)
|
|
config = QwenImage21VAEConfig(arch_config=ac)
|
|
config.load_encoder = False
|
|
vae = AutoencoderKLQwenImage21(config).cuda().bfloat16().eval()
|
|
torch.manual_seed(0)
|
|
with torch.no_grad():
|
|
for name, param in vae.named_parameters():
|
|
if name.endswith("gamma"):
|
|
param.copy_(1 + 0.1 * torch.randn_like(param))
|
|
else:
|
|
torch.nn.init.normal_(param, std=0.1)
|
|
return vae
|
|
|
|
|
|
def _qwen21_latent(size=4):
|
|
return torch.randn(1, 64, 1, size, size, device="cuda", dtype=torch.bfloat16)
|
|
|
|
|
|
def _modules_of(vae, cls):
|
|
return [m for m in vae.modules() if isinstance(m, cls)]
|
|
|
|
|
|
def _first_divergence(reference, optimized, first_z, z):
|
|
"""Where the optimized decoder first departs from the eager one on ``z``, and which fast path is responsible.
|
|
|
|
Both models keep the original module names, so hooks on the residual /
|
|
up blocks and the output conv line up one to one (the wrapped norms are
|
|
skipped: the optimized one includes the SiLU). Then fresh optimized copies
|
|
take their first sight on ``first_z`` with one fast-path family disabled
|
|
at a time and decode ``z``, so a mismatch names its family without an
|
|
interactive session.
|
|
"""
|
|
watched = (
|
|
"QwenImage21ResidualBlock",
|
|
"QwenImage21ResidualUpBlock",
|
|
"QwenImage21MidBlock",
|
|
)
|
|
outputs = {"reference": [], "optimized": []}
|
|
handles = []
|
|
for tag, model in (("reference", reference), ("optimized", optimized)):
|
|
for name, m in model.named_modules():
|
|
if type(m).__name__ in watched or name == "decoder.conv_out":
|
|
handles.append(
|
|
m.register_forward_hook(
|
|
lambda mod, inp, out, tag=tag, name=name: outputs[tag].append(
|
|
(name, out.detach().clone())
|
|
)
|
|
)
|
|
)
|
|
try:
|
|
ref_out, opt_out = reference.decode(z), optimized.decode(z)
|
|
finally:
|
|
for h in handles:
|
|
h.remove()
|
|
blocks = []
|
|
for (name, ref), (_, opt) in zip(
|
|
outputs["reference"], outputs["optimized"], strict=True
|
|
):
|
|
if ref.shape != opt.shape or not torch.equal(ref, opt):
|
|
diff = (ref.float() - opt.float()).abs()
|
|
blocks.append(
|
|
f"{name}: max|diff| {diff.max().item():.3e}, "
|
|
f"{(diff > 0).sum().item()}/{diff.numel()} elements"
|
|
)
|
|
if len(blocks) == 3:
|
|
break
|
|
|
|
def decode_without(family):
|
|
model = qwen21_vae_opt.maybe_optimize_qwen_image21_vae(deepcopy(reference))
|
|
saved = {
|
|
name: getattr(qwen21_vae_opt, name)
|
|
for name in ("_BIAS_RESIDUAL_FUSION", "_UPSAMPLE_BIAS_FUSION")
|
|
}
|
|
try:
|
|
for name in saved:
|
|
gate = qwen21_vae_opt.BitExactFusionGate(f"isolation {name}")
|
|
if family == name:
|
|
gate.disable()
|
|
setattr(qwen21_vae_opt, name, gate)
|
|
for m in model.modules():
|
|
if family == "padding fold" and isinstance(
|
|
m, qwen21_vae_opt.FoldedPadConv2d
|
|
):
|
|
m._gates["nchw"].disable()
|
|
m._gates["nhwc"].disable()
|
|
if family == "norm+SiLU tail" and isinstance(
|
|
m, qwen21_vae_opt.FusedChannelRMSNormSiLU
|
|
):
|
|
m._exact_gate.disable()
|
|
model.decode(first_z) # first sight for the remaining families
|
|
return torch.equal(model.decode(z), ref_out)
|
|
finally:
|
|
for name, gate in saved.items():
|
|
setattr(qwen21_vae_opt, name, gate)
|
|
|
|
families = (
|
|
"padding fold",
|
|
"norm+SiLU tail",
|
|
"_BIAS_RESIDUAL_FUSION",
|
|
"_UPSAMPLE_BIAS_FUSION",
|
|
)
|
|
isolation = ", ".join(
|
|
f"without {family}: {'equal' if decode_without(family) else 'differs'}"
|
|
for family in families
|
|
)
|
|
return (
|
|
"decode "
|
|
+ ("equal" if torch.equal(ref_out, opt_out) else "differs")
|
|
+ f"; eager repeats on this latent: {torch.equal(reference.decode(z), ref_out)}"
|
|
+ f"; optimized repeats on this latent: {torch.equal(optimized.decode(z), opt_out)}"
|
|
+ "; first divergent blocks: "
|
|
+ ("; ".join(blocks) if blocks else "none")
|
|
+ "; "
|
|
+ isolation
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def qwen21_deterministic_cudnn():
|
|
# Bit-exactness is asserted against the eager chain, so both sides must run
|
|
# deterministic conv algorithms; cuDNN may otherwise pick split-K engines
|
|
# whose atomics reorder the sum between two calls.
|
|
previous = torch.backends.cudnn.deterministic
|
|
torch.backends.cudnn.deterministic = True
|
|
try:
|
|
yield
|
|
finally:
|
|
torch.backends.cudnn.deterministic = previous
|
|
|
|
|
|
@pytest.fixture
|
|
def qwen21_vae_gates(monkeypatch, qwen21_deterministic_cudnn):
|
|
"""Fresh copies of the module-level first-sight gates, so tests do not share state."""
|
|
gates = {
|
|
"_BIAS_RESIDUAL_FUSION": qwen21_vae_opt.BitExactFusionGate(
|
|
"test bias + residual"
|
|
),
|
|
"_UPSAMPLE_BIAS_FUSION": qwen21_vae_opt.BitExactFusionGate(
|
|
"test upsampler bias"
|
|
),
|
|
}
|
|
for name, gate in gates.items():
|
|
monkeypatch.setattr(qwen21_vae_opt, name, gate)
|
|
return gates
|
|
|
|
|
|
@requires_inline_ptx
|
|
@torch.no_grad()
|
|
def test_qwen21_vae_lossless_decode_is_bit_identical(qwen21_vae_gates):
|
|
reference = _qwen21_vae()
|
|
optimized = qwen21_vae_opt.maybe_optimize_qwen_image21_vae(deepcopy(reference))
|
|
# wrappers keep parameter names and values
|
|
assert set(optimized.state_dict()) == set(reference.state_dict())
|
|
for key, value in optimized.state_dict().items():
|
|
assert torch.equal(value, reference.state_dict()[key])
|
|
|
|
# 64x64 latent (1024x1024 output): every fold site is above the size floor.
|
|
z = _qwen21_latent(64)
|
|
expected = reference.decode(z)
|
|
assert torch.equal(reference.decode(z), expected), (
|
|
"the eager decode itself is not reproducible on this platform"
|
|
)
|
|
assert torch.equal(optimized.decode(z), expected)
|
|
|
|
# Kernels that are exact by construction must have engaged.
|
|
norm_gates = [
|
|
m._exact_gate
|
|
for m in _modules_of(optimized, qwen21_vae_opt.FusedChannelRMSNormSiLU)
|
|
]
|
|
assert len(norm_gates) >= 8
|
|
for gate in norm_gates + list(qwen21_vae_gates.values()):
|
|
assert gate.verified and not gate.disabled, gate.name
|
|
# The padding fold is kept only where cuDNN's kernel for the folded
|
|
# descriptor reproduces the padded conv bit for bit (true on SM120, not on
|
|
# H100), so a declined fold is a valid outcome; each conv must have decided.
|
|
fold_gates = [
|
|
m._gates["nchw"] for m in _modules_of(optimized, qwen21_vae_opt.FoldedPadConv2d)
|
|
]
|
|
assert len(fold_gates) >= 8
|
|
assert all(gate.verified or gate.disabled for gate in fold_gates)
|
|
|
|
z2 = _qwen21_latent(64)
|
|
assert torch.equal(optimized.decode(z2), reference.decode(z2)), _first_divergence(
|
|
reference, optimized, z, z2
|
|
)
|
|
|
|
|
|
@requires_inline_ptx
|
|
@torch.no_grad()
|
|
def test_qwen21_vae_mismatch_falls_back_to_the_eager_chain(
|
|
qwen21_vae_gates, monkeypatch
|
|
):
|
|
reference = _qwen21_vae()
|
|
optimized = qwen21_vae_opt.maybe_optimize_qwen_image21_vae(deepcopy(reference))
|
|
monkeypatch.setattr(
|
|
qwen21_vae_opt,
|
|
"channel_rmsnorm_finish_silu",
|
|
lambda x, norm, gamma, scale: torch.zeros_like(x),
|
|
)
|
|
monkeypatch.setattr(
|
|
qwen21_vae_opt, "bias_residual_add", lambda y, bias, h: torch.zeros_like(y)
|
|
)
|
|
z = _qwen21_latent()
|
|
assert torch.equal(optimized.decode(z), reference.decode(z))
|
|
norm_gates = [
|
|
m._exact_gate
|
|
for m in _modules_of(optimized, qwen21_vae_opt.FusedChannelRMSNormSiLU)
|
|
]
|
|
assert norm_gates and all(g.disabled and not g.verified for g in norm_gates)
|
|
bias_gate = qwen21_vae_gates["_BIAS_RESIDUAL_FUSION"]
|
|
assert bias_gate.disabled and not bias_gate.verified
|
|
|
|
|
|
@requires_inline_ptx
|
|
@torch.no_grad()
|
|
def test_qwen21_vae_extra_high_decode_is_close_and_lossless_is_restored(
|
|
qwen21_vae_gates,
|
|
):
|
|
reference = _qwen21_vae()
|
|
optimized = qwen21_vae_opt.maybe_optimize_qwen_image21_vae(deepcopy(reference))
|
|
upsamplers = [
|
|
m
|
|
for m in _modules_of(optimized, QwenImage21Resample)
|
|
if m.mode.startswith("upsample")
|
|
]
|
|
assert len(upsamplers) == 4
|
|
assert all(
|
|
isinstance(m.resample, qwen21_vae_opt.FusedUpsample2xConv) for m in upsamplers
|
|
)
|
|
|
|
z = _qwen21_latent()
|
|
expected = reference.decode(z)
|
|
with use_vae_fast_path(optimized, True):
|
|
fast = optimized.decode(z)
|
|
assert optimized.decoder._sgl_channels_last
|
|
torch.testing.assert_close(fast.float(), expected.float(), atol=0.05, rtol=0)
|
|
# gate off: layout and kernels revert, output is bit-identical again
|
|
assert torch.equal(optimized.decode(z), expected)
|
|
assert not optimized.decoder._sgl_channels_last
|
|
|
|
|
|
@requires_inline_ptx
|
|
@torch.no_grad()
|
|
def test_qwen21_vae_folded_upsampler_follows_the_weights_and_moves(qwen21_vae_gates):
|
|
reference = _qwen21_vae()
|
|
optimized = qwen21_vae_opt.maybe_optimize_qwen_image21_vae(deepcopy(reference))
|
|
z = _qwen21_latent()
|
|
with use_vae_fast_path(optimized, True):
|
|
before = optimized.decode(z) # builds the folded kernels
|
|
for model in (reference, optimized):
|
|
for m in _modules_of(model, QwenImage21Resample):
|
|
if m.mode.startswith("upsample"):
|
|
m.resample[1].weight.zero_()
|
|
m.resample[1].bias.zero_()
|
|
expected = reference.decode(z)
|
|
assert not torch.equal(expected, before)
|
|
with use_vae_fast_path(optimized, True):
|
|
fast = optimized.decode(z)
|
|
torch.testing.assert_close(fast.float(), expected.float(), atol=0.05, rtol=0)
|
|
|
|
# CPU offload must not leave the folded kernels, or anything else, on the GPU.
|
|
optimized.to("cpu")
|
|
for m in optimized.modules():
|
|
for value in list(vars(m).values()) + list(m._buffers.values()):
|
|
assert not (isinstance(value, torch.Tensor) and value.is_cuda), type(m)
|
|
optimized.to("cuda")
|
|
with use_vae_fast_path(optimized, True):
|
|
torch.testing.assert_close(
|
|
optimized.decode(z).float(), expected.float(), atol=0.05, rtol=0
|
|
)
|
|
|
|
|
|
def test_qwen21_vae_fold_rejects_convs_it_cannot_express():
|
|
good = torch.nn.Conv2d(4, 6, 3, padding=1)
|
|
assert fold_upsample2x_conv2d_weight(good).shape == (4, 6, 4, 4)
|
|
for bad in (
|
|
torch.nn.Conv2d(4, 6, 1),
|
|
torch.nn.Conv2d(4, 6, 3, padding=1, stride=2),
|
|
torch.nn.Conv2d(4, 4, 3, padding=1, groups=2),
|
|
torch.nn.Conv2d(4, 6, 3, padding=1, padding_mode="reflect"),
|
|
):
|
|
with pytest.raises(ValueError):
|
|
fold_upsample2x_conv2d_weight(bad)
|
|
|
|
|
|
def test_qwen21_vae_optimizer_passes_other_vaes_through():
|
|
assert (
|
|
qwen21_vae_opt.maybe_optimize_qwen_image21_vae(torch.nn.Linear(2, 2))
|
|
is not None
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(torch.version.hip is not None, reason="NVIDIA CUDA required")
|
|
def test_anima_rope_dispatch_and_fallback():
|
|
import sglang.kernels.ops.diffusion as ops
|
|
import sglang.multimodal_gen.runtime.models.dits.anima as anima
|
|
|
|
generator = torch.Generator(device="cuda").manual_seed(42)
|
|
q, k = [
|
|
torch.randn(
|
|
2, 17, 3, 128, device="cuda", dtype=torch.bfloat16, generator=generator
|
|
)
|
|
for _ in range(2)
|
|
]
|
|
angles = torch.randn(17, 128, device="cuda", generator=generator)
|
|
cos, sin = angles.cos(), angles.sin()
|
|
args = q, k, cos, sin
|
|
expected = anima._anima_rope_eager(*args)
|
|
expected_fp32 = anima._anima_rope_eager(q.float(), k.float(), cos, sin)
|
|
gate = BitExactFusionGate("Anima test", per_signature=True)
|
|
with (
|
|
patch.object(anima, "_ANIMA_ROPE", gate),
|
|
patch.object(
|
|
ops, "fused_rope_rotate_half_fp32", wraps=ops.fused_rope_rotate_half_fp32
|
|
) as fused,
|
|
patch.object(
|
|
anima, "_anima_rope_eager", wraps=anima._anima_rope_eager
|
|
) as eager,
|
|
):
|
|
for _ in range(2):
|
|
outputs = anima._anima_rope(*args)
|
|
assert all(torch.equal(out, ref) for out, ref in zip(outputs, expected))
|
|
assert fused.call_count == 2
|
|
assert eager.call_count == 1 # Only the first call verifies against eager.
|
|
assert gate.is_verified((q.device, q.dtype, q.shape)) and not gate.disabled
|
|
|
|
fused.reset_mock()
|
|
outputs = anima._anima_rope(q.float(), k.float(), cos, sin)
|
|
fused.assert_not_called()
|
|
assert eager.call_count == 2
|
|
assert all(torch.equal(out, ref) for out, ref in zip(outputs, expected_fp32))
|
|
|
|
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA PTX")
|
|
@pytest.mark.parametrize("packed", [False, True])
|
|
def test_moss_image_situ_fusion_is_bit_exact_and_verified(monkeypatch, packed):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
gate_state = BitExactFusionGate("MOSSImage SiTU test")
|
|
monkeypatch.setattr(moss_image, "_SITU_FUSION", gate_state)
|
|
torch.manual_seed(0)
|
|
if packed:
|
|
projection = torch.randn(1, 19, 256, device="cuda", dtype=torch.bfloat16)
|
|
hidden, gate = projection.chunk(2, -1)
|
|
else:
|
|
hidden = torch.randn(1, 19, 128, device="cuda", dtype=torch.bfloat16)
|
|
gate = torch.randn_like(hidden)
|
|
expected = moss_image._situ_native(hidden, gate)
|
|
for _ in range(2):
|
|
assert torch.equal(moss_image._situ(hidden, gate), expected)
|
|
assert gate_state.verified and not gate_state.disabled
|
|
|
|
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA")
|
|
def test_moss_image_rope_fusion_is_bit_exact_and_verified(monkeypatch):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
gate_state = BitExactFusionGate("MOSSImage RoPE test")
|
|
monkeypatch.setattr(moss_image, "_ROPE_FUSION", gate_state)
|
|
torch.manual_seed(0)
|
|
x = torch.randn(1, 17, 2, 128, device="cuda", dtype=torch.bfloat16)
|
|
rope = moss_image._RoPE(128, 10000).cuda()(2, 4, 9, torch.device("cuda"))
|
|
rope = moss_image._prepare_rope(rope.to(x.dtype))
|
|
expected = moss_image._rotate_native(x, rope)
|
|
for _ in range(2):
|
|
assert torch.equal(moss_image._rotate(x, rope), expected)
|
|
assert gate_state.verified and not gate_state.disabled
|
|
|
|
|
|
def test_moss_image_situ_fusion_mismatch_disables_fast_path(monkeypatch):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
gate_state = BitExactFusionGate("test SiTU")
|
|
monkeypatch.setattr(moss_image, "_SITU_FUSION", gate_state)
|
|
monkeypatch.setattr(
|
|
moss_image, "can_use_fused_situ_softcap_mul", lambda **kwargs: True
|
|
)
|
|
calls = []
|
|
|
|
def incorrect(hidden, gate):
|
|
calls.append(1)
|
|
return torch.zeros_like(hidden)
|
|
|
|
monkeypatch.setattr(moss_image, "fused_situ_softcap_mul_bitexact", incorrect)
|
|
hidden = torch.ones(1, 2, 4)
|
|
gate = torch.full_like(hidden, 2)
|
|
expected = moss_image._situ_native(hidden, gate)
|
|
for _ in range(2):
|
|
assert torch.equal(moss_image._situ(hidden, gate), expected)
|
|
assert gate_state.disabled and not gate_state.verified
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_moss_image_rope_fusion_mismatch_disables_fast_path(monkeypatch):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
gate_state = BitExactFusionGate("test RoPE")
|
|
monkeypatch.setattr(moss_image, "_ROPE_FUSION", gate_state)
|
|
calls = []
|
|
|
|
def incorrect(x, cos, sin):
|
|
calls.append(1)
|
|
return torch.zeros_like(x)
|
|
|
|
monkeypatch.setattr(moss_image, "fused_rope_rotate_half_bitexact", incorrect)
|
|
x = torch.ones(1, 3, 2, 16, device="cuda", dtype=torch.bfloat16)
|
|
rope = tuple(
|
|
torch.full((1, 3, 16), value, device=x.device, dtype=x.dtype)
|
|
for value in (0, 1)
|
|
)
|
|
for _ in range(2):
|
|
assert torch.equal(moss_image._rotate(x, rope), x)
|
|
assert gate_state.disabled and not gate_state.verified
|
|
assert len(calls) == 1
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA PTX")
|
|
@pytest.mark.parametrize("shape", [(1, 4096, 3072), (1, 97, 24, 128)])
|
|
def test_moss_image_rmsnorm_fusion_is_bit_exact_and_verified(monkeypatch, shape):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
state = BitExactFusionGate("MOSSImage RMSNorm test", per_signature=True)
|
|
monkeypatch.setattr(moss_image, "_RMSNORM_FUSION", state)
|
|
norm = moss_image._RMSNorm(shape[-1]).to("cuda", torch.bfloat16)
|
|
norm.weight.copy_(torch.randn_like(norm.weight))
|
|
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
|
|
expected = norm.forward_native(x)
|
|
for _ in range(2):
|
|
assert torch.equal(norm(x), expected)
|
|
assert state.verified and not state.disabled
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["mismatch", "exception"])
|
|
def test_moss_image_rmsnorm_failure_disables_fast_path(monkeypatch, failure):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
state = BitExactFusionGate("MOSSImage RMSNorm test", per_signature=True)
|
|
monkeypatch.setattr(moss_image, "_RMSNORM_FUSION", state)
|
|
monkeypatch.setattr(
|
|
moss_image, "can_use_rmsnorm_cast_before_weight", lambda **kwargs: True
|
|
)
|
|
calls = []
|
|
|
|
def incorrect(x, weight):
|
|
calls.append(1)
|
|
if failure == "exception":
|
|
raise RuntimeError("kernel unavailable")
|
|
return torch.zeros_like(x)
|
|
|
|
monkeypatch.setattr(moss_image, "rmsnorm_cast_before_weight_bitexact", incorrect)
|
|
norm = moss_image._RMSNorm(128)
|
|
x = torch.ones(1, 3, 128)
|
|
for _ in range(2):
|
|
assert torch.equal(norm(x), norm.forward_native(x))
|
|
assert state.disabled and not state.verified and len(calls) == 1
|
|
|
|
|
|
@torch.no_grad()
|
|
def test_moss_image_rmsnorm_unsupported_layout_uses_native(monkeypatch):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
state = BitExactFusionGate("MOSSImage RMSNorm test", per_signature=True)
|
|
monkeypatch.setattr(moss_image, "_RMSNORM_FUSION", state)
|
|
norm = moss_image._RMSNorm(128).to("cuda", torch.bfloat16)
|
|
x = torch.randn(2, 7, 3 * 128, device="cuda", dtype=torch.bfloat16)[..., :128]
|
|
assert torch.equal(norm(x), norm.forward_native(x))
|
|
assert not state.verified and not state.disabled
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA JIT")
|
|
@pytest.mark.parametrize("batch", [1, 2])
|
|
def test_moss_image_block_modulation_and_residual_match_eager(monkeypatch, batch):
|
|
from sglang.kernels.ops.diffusion import (
|
|
can_use_modulate_scale_shift_cuda,
|
|
can_use_residual_gate_add_cuda,
|
|
modulate_scale_shift,
|
|
residual_gate_add,
|
|
)
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
class ValueAttention(nn.Module):
|
|
def forward(self, q, k, v):
|
|
return v
|
|
|
|
monkeypatch.setattr(moss_image, "LocalAttention", lambda **kwargs: ValueAttention())
|
|
block = moss_image._Block(128, 256, 1).to("cuda", torch.bfloat16)
|
|
block.attention = ValueAttention()
|
|
x = torch.randn(batch, 19, 128, device="cuda", dtype=torch.bfloat16)
|
|
modulation = torch.randn(batch, 6, 128, device="cuda", dtype=x.dtype)
|
|
rope = (
|
|
torch.zeros(batch, 19, 128, device="cuda", dtype=x.dtype),
|
|
torch.ones(batch, 19, 128, device="cuda", dtype=x.dtype),
|
|
)
|
|
calls = []
|
|
|
|
def modulate(x, scale, shift):
|
|
calls.append(("modulate", can_use_modulate_scale_shift_cuda(x, scale, shift)))
|
|
return modulate_scale_shift(x, scale, shift)
|
|
|
|
def residual(x, update, gate):
|
|
calls.append(("residual", can_use_residual_gate_add_cuda(x, update, gate)))
|
|
return residual_gate_add(x, update, gate)
|
|
|
|
monkeypatch.setattr(moss_image, "modulate_scale_shift", modulate)
|
|
monkeypatch.setattr(moss_image, "residual_gate_add", residual)
|
|
actual = block(x, rope, modulation)
|
|
monkeypatch.setattr(
|
|
moss_image,
|
|
"modulate_scale_shift",
|
|
lambda x, s, b: x * (1 + s[:, None]) + b[:, None],
|
|
)
|
|
monkeypatch.setattr(moss_image, "residual_gate_add", lambda x, h, g: x + h * g)
|
|
expected = block(x, rope, modulation)
|
|
assert torch.equal(actual, expected)
|
|
assert calls == [("modulate", batch == 1), ("residual", True)] * 2
|
|
|
|
|
|
@torch.no_grad()
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA PTX and graph capture")
|
|
def test_moss_image_rmsnorm_verifies_each_regime_and_preserves_graph_capture(
|
|
monkeypatch,
|
|
):
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
state = BitExactFusionGate("MOSSImage RMSNorm test", per_signature=True)
|
|
monkeypatch.setattr(moss_image, "_RMSNORM_FUSION", state)
|
|
token_norm = moss_image._RMSNorm(3072).to("cuda", torch.bfloat16)
|
|
for rows in [13, 17]:
|
|
x = torch.randn(1, rows, 3072, device="cuda", dtype=torch.bfloat16)
|
|
assert torch.equal(token_norm(x), token_norm.forward_native(x))
|
|
assert len(state.verified_sigs) == 2
|
|
|
|
head_norm = moss_image._RMSNorm(128).to("cuda", torch.bfloat16)
|
|
x = torch.randn(1, 19, 128, device="cuda", dtype=torch.bfloat16)
|
|
stream = torch.cuda.Stream()
|
|
stream.wait_stream(torch.cuda.current_stream())
|
|
with torch.cuda.stream(stream):
|
|
for _ in range(3):
|
|
head_norm.forward_native(x)
|
|
stream.synchronize()
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph, stream=stream):
|
|
captured = head_norm(x)
|
|
graph.replay()
|
|
assert torch.equal(captured, head_norm.forward_native(x))
|
|
# A previously verified width must not permit a host-synchronizing check
|
|
# for a new width inside capture.
|
|
assert len(state.verified_sigs) == 2
|
|
|
|
assert torch.equal(head_norm(x), head_norm.forward_native(x))
|
|
assert len(state.verified_sigs) == 3 and not state.disabled
|
|
stream.wait_stream(torch.cuda.current_stream())
|
|
fused_graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(fused_graph, stream=stream):
|
|
fused = head_norm(x)
|
|
for _ in range(2):
|
|
x.copy_(torch.randn_like(x))
|
|
fused_graph.replay()
|
|
assert torch.equal(fused, head_norm.forward_native(x))
|
|
|
|
|
|
@pytest.mark.skipif(torch.version.hip is not None, reason="Requires CUDA PTX")
|
|
@pytest.mark.parametrize("epilogue", ["modulate", "rope"])
|
|
def test_moss_image_norm_epilogue_gate_and_graph_replay(monkeypatch, epilogue):
|
|
import sglang.multimodal_gen.runtime.models.dits.moss_image as moss_image
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
|
|
gate = BitExactFusionGate("test norm epilogue")
|
|
if epilogue == "modulate":
|
|
monkeypatch.setattr(moss_image, "_NORM_MOD_FUSION", gate)
|
|
norm = moss_image._RMSNorm(3072).cuda().bfloat16()
|
|
x = torch.randn(1, 33, 3072, device="cuda", dtype=torch.bfloat16)
|
|
scale = torch.randn(1, 3072, device="cuda", dtype=torch.bfloat16)
|
|
shift = torch.randn_like(scale)
|
|
call = lambda: moss_image._norm_modulate(norm, x, scale, shift)
|
|
reference = lambda: (
|
|
norm.forward_native(x) * (1 + scale[:, None]) + shift[:, None]
|
|
)
|
|
else:
|
|
monkeypatch.setattr(moss_image, "_NORM_ROPE_FUSION", gate)
|
|
norm = moss_image._RMSNorm(128).cuda().bfloat16()
|
|
x = torch.randn(1, 33, 3072, device="cuda", dtype=torch.bfloat16)
|
|
rope = tuple(
|
|
torch.randn(1, 33, 128, device="cuda", dtype=torch.bfloat16)
|
|
for _ in range(2)
|
|
)
|
|
call = lambda: moss_image._norm_rope(norm, x, rope, 24)
|
|
reference = lambda: moss_image._rotate_native(
|
|
norm.forward_native(x.unflatten(-1, (24, 128))), rope
|
|
).flatten(2)
|
|
with torch.no_grad():
|
|
# The unverified graph must capture the existing fallback without a sync.
|
|
cold = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(cold):
|
|
cold_out = call()
|
|
assert not gate.verified
|
|
assert torch.equal(call(), reference())
|
|
assert gate.verified and not gate.disabled
|
|
hot = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(hot):
|
|
hot_out = call()
|
|
for _ in range(2):
|
|
x.copy_(torch.randn_like(x))
|
|
cold.replay()
|
|
hot.replay()
|
|
expected = reference()
|
|
assert torch.equal(cold_out, expected)
|
|
assert torch.equal(hot_out, expected)
|
|
|
|
|
|
@pytest.mark.skipif(not is_cuda(), reason="Requires CUDA PTX")
|
|
@pytest.mark.parametrize("operation", ["situ", "rmsnorm"])
|
|
@torch.no_grad()
|
|
def test_moss_image_selected_kernel_rejects_mismatched_operands(monkeypatch, operation):
|
|
"""Invalid operands must not silently disable a selected fusion and run eager."""
|
|
from sglang.kernels.ops.diffusion import BitExactFusionGate
|
|
from sglang.multimodal_gen.runtime.models.dits import moss_image
|
|
|
|
x = torch.ones(1, 17, 128, device="cuda", dtype=torch.bfloat16)
|
|
if operation == "situ":
|
|
monkeypatch.setattr(
|
|
moss_image,
|
|
"_SITU_FUSION",
|
|
BitExactFusionGate("MOSSImage invalid input test"),
|
|
)
|
|
with pytest.raises(ValueError):
|
|
moss_image._situ(x, x[:, :-1])
|
|
assert not moss_image._SITU_FUSION.disabled
|
|
else:
|
|
monkeypatch.setattr(
|
|
moss_image,
|
|
"_RMSNORM_FUSION",
|
|
BitExactFusionGate("MOSSImage invalid input test"),
|
|
)
|
|
norm = moss_image._RMSNorm(64).to(device="cuda", dtype=torch.bfloat16)
|
|
with pytest.raises(ValueError):
|
|
norm(x)
|
|
assert not moss_image._RMSNORM_FUSION.disabled
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__, "-v"]))
|