Files
sglang/test/registered/kernels/ops/diffusion/test_sites.py
T

598 lines
22 KiB
Python

"""``diffusion.sites``: the two gate protocols and the mount lifecycle.
Nothing here is a kernel. ``sites`` decides *whether* a fusion is allowed to
run for a given request and model:
- :class:`QualityGatedFusion` -- for fusions that are **not** bit-exact. They
mount onto marked ``nn.Module`` sites only for ``quality="high"``, at batch
boundaries, all-or-nothing per transformer.
- :class:`BitExactFusionGate` -- for fusions that **are** bit-exact. They run
by default but verify themselves against the live eager chain on first
sight and disable permanently on any mismatch.
The protocol tests are pure-CPU. The mount-lifecycle tests below use
synthetic sites (a bare ``nn.Module`` with the marker attribute) so they test
the protocol rather than any one model; the real model wrappers live in
``test_model_fast_paths.py``.
"""
import sys
from types import ModuleType
from unittest.mock import MagicMock, patch
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
import sglang.kernels.ops.diffusion.sites.fused_gate_rmsnorm_site as gate_rmsnorm
import sglang.kernels.ops.diffusion.sites.fused_linear_gelu_site as linear_gelu
import sglang.kernels.ops.diffusion.sites.lingbot_video_rmsnorm_site as lingbot_video_rmsnorm
import sglang.kernels.ops.diffusion.sites.qwen_image_added_qkv_site as qwen_image_added_qkv
import sglang.kernels.ops.diffusion.sites.sana_video_linear_attention_site as sana_video_linear_attention
from sglang.kernels.ops.diffusion import (
FLUX2_NVFP4_SWIGLU_QUANT,
FUSED_LN_MODULATE,
BitExactFusionGate,
QualityGatedFusion,
can_use_ln_modulate,
flashinfer_rmsnorm_diagnostic_hint,
fused_ln_modulate,
tensors_equal,
)
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
register_cuda_ci(est_time=38, stage="base-b-kernel-unit", runner_config="1-gpu-large")
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="CUDA required"
)
# ---------------------------------------------------------------------------
# QualityGatedFusion protocol (CPU)
# ---------------------------------------------------------------------------
def test_quality_gate_mounts_and_unmounts_all_sites():
fusion = QualityGatedFusion(
name="test fusion",
marker_attr="_test_fusion_site",
enabled_attr="_test_fusion_enabled",
)
root = nn.ModuleList([nn.Module(), nn.Module()])
for index, site in enumerate(root):
fusion.mark(site, index)
assert [fusion.metadata(site) for site in root] == [0, 1]
assert fusion.mount(root)
assert all(fusion.is_enabled(site) for site in root)
fusion.unmount(root)
assert not any(fusion.is_enabled(site) for site in root)
def test_quality_gate_rejection_is_all_or_nothing():
fusion = QualityGatedFusion(
name="test fusion",
marker_attr="_test_fusion_site",
enabled_attr="_test_fusion_enabled",
reject_reason=lambda site: "rejected" if fusion.metadata(site) else None,
)
root = nn.ModuleList([nn.Module(), nn.Module()])
for index, site in enumerate(root):
fusion.mark(site, index)
assert not fusion.mount(root)
assert not any(fusion.is_enabled(site) for site in root)
assert not fusion.mount(nn.Module())
def test_flux2_nvfp4_swiglu_quant_is_disabled_until_quality_gate_mounts():
site = nn.Module()
root = nn.ModuleList([site])
FLUX2_NVFP4_SWIGLU_QUANT.mark(site)
assert not FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site)
assert FLUX2_NVFP4_SWIGLU_QUANT.mount(root)
assert FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site)
FLUX2_NVFP4_SWIGLU_QUANT.unmount(root)
assert not FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site)
def test_qwen_image_added_qkv_site_is_request_scoped():
site = nn.Module()
site.to_added_qkv = nn.Module()
site.to_added_qkv.quant_config = None
site.to_added_qkv.output_partition_sizes = [8, 8, 8]
qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.mark(site)
assert not qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site)
assert qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.mount(site)
assert qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site)
qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.unmount(site)
assert not qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site)
# -------------------------------------------------------------------------
# BitExactFusionGate protocol (CPU)
# -------------------------------------------------------------------------
def test_bitexact_gate_once_mode_verifies_then_reuses():
gate = BitExactFusionGate("once")
calls = {"fused": 0, "ref": 0}
def fused():
calls["fused"] += 1
return torch.tensor([1.0])
def ref():
calls["ref"] += 1
return torch.tensor([1.0])
assert torch.equal(gate.accept_or_fallback(fused(), ref()), torch.tensor([1.0]))
assert gate.verified and not gate.disabled and calls == {"fused": 1, "ref": 1}
assert torch.equal(fused(), torch.tensor([1.0]))
assert calls == {"fused": 2, "ref": 1}
def test_bitexact_gate_mismatch_disables_permanently():
gate = BitExactFusionGate("mismatch")
out = gate.accept_or_fallback(
torch.tensor([1.0]),
torch.tensor([2.0]),
)
assert torch.equal(out, torch.tensor([2.0]))
assert gate.disabled and not gate.verified
def test_bitexact_gate_per_signature_tracks_each_sig():
gate = BitExactFusionGate("sig", per_signature=True)
a = torch.tensor([1.0])
assert torch.equal(gate.accept_or_fallback(a, a, sig=("a",)), a)
assert gate.is_verified(("a",))
assert not gate.is_verified(("b",))
assert torch.equal(gate.accept_or_fallback(a, a, sig=("b",)), a)
assert gate.verified_sigs == {("a",), ("b",)}
def test_bitexact_gate_skips_first_sight_during_graph_capture(monkeypatch):
# Negative-branch contract: an unverified gate must not attempt first-sight
# verification inside CUDA graph capture — the eager-reference host sync
# would abort the capture (and BCG would permanently block the signature).
gate = BitExactFusionGate("capture")
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
assert not gate.can_attempt_once()
# A verified gate replays the fused kernel alone, which is capture-safe.
gate.mark_verified()
assert gate.can_attempt_once()
def test_bitexact_gate_run_verifies_each_signature_outside_capture(monkeypatch):
# A new signature must not verify (eager reference + host sync) inside
# capture, while a verified one replays the fused kernel alone there.
gate = BitExactFusionGate("run", per_signature=True)
calls = []
def fused():
calls.append("fused")
return torch.tensor([1.0])
def ref():
calls.append("ref")
return torch.tensor([1.0])
def capturing(active):
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: active)
calls.clear()
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
capturing(True)
gate.run(fused, ref, sig="a")
assert calls == ["ref"] and not gate.is_verified("a")
capturing(False)
gate.run(fused, ref, sig="a")
assert calls == ["fused", "ref"] and gate.is_verified("a")
capturing(True)
gate.run(fused, ref, sig="a")
assert calls == ["fused"]
gate.run(fused, ref, sig="b")
assert calls == ["fused", "ref"] and not gate.is_verified("b")
def invalid():
raise ValueError("bad operands")
with pytest.raises(ValueError):
gate.run(invalid, ref, sig="a", reraise=(ValueError,))
assert not gate.disabled
def test_tensors_equal_supports_sequences():
assert tensors_equal(
(torch.tensor([1.0]), torch.tensor([2.0])),
(torch.tensor([1.0]), torch.tensor([2.0])),
)
assert not tensors_equal(
(torch.tensor([1.0]), torch.tensor([2.0])),
(torch.tensor([1.0]), torch.tensor([3.0])),
)
class TestBitExactFallbackDiagnostics(CustomTestCase):
def test_mismatch_warning_is_actionable_and_diagnostic_is_lazy(self):
logger = MagicMock()
diagnostic = MagicMock(return_value="backend=CuTe DSL")
gate = BitExactFusionGate("diagnostic")
matched = gate.accept_or_fallback(
torch.tensor([1.0]),
torch.tensor([1.0]),
logger=logger,
diagnostic_hint=diagnostic,
)
self.assertTrue(torch.equal(matched, torch.tensor([1.0])))
diagnostic.assert_not_called()
logger.warning_once.assert_not_called()
gate = BitExactFusionGate("diagnostic")
fallback = gate.accept_or_fallback(
torch.tensor([1.0]),
torch.tensor([2.0]),
logger=logger,
diagnostic_hint=diagnostic,
)
self.assertTrue(torch.equal(fallback, torch.tensor([2.0])))
diagnostic.assert_called_once_with()
warning = logger.warning_once.call_args.args[0]
self.assertIn("Correctness is preserved", warning)
self.assertIn("reference kernel or reduction-order change", warning)
self.assertIn("backend=CuTe DSL", warning)
def test_diagnostic_failure_cannot_break_the_eager_fallback(self):
logger = MagicMock()
def broken_diagnostic():
raise RuntimeError("diagnostics unavailable")
gate = BitExactFusionGate("diagnostic")
fallback = gate.accept_or_fallback(
torch.tensor([1.0]),
torch.tensor([2.0]),
logger=logger,
diagnostic_hint=broken_diagnostic,
)
self.assertTrue(torch.equal(fallback, torch.tensor([2.0])))
self.assertTrue(gate.disabled)
self.assertIn("Correctness is preserved", logger.warning_once.call_args.args[0])
def test_flashinfer_rmsnorm_hint_reports_backend_and_versions(self):
flashinfer = ModuleType("flashinfer")
flashinfer_norm = ModuleType("flashinfer.norm")
flashinfer_norm._USE_CUDA_NORM = False
versions = {
"flashinfer-python": "0.6.12",
"flashinfer-cubin": "0.6.12",
"flashinfer-jit-cache": "0.6.12+cu130",
}
with (
patch.dict(
sys.modules,
{"flashinfer": flashinfer, "flashinfer.norm": flashinfer_norm},
),
patch("importlib.metadata.version", side_effect=versions.__getitem__),
patch.dict("os.environ", {"FLASHINFER_USE_CUDA_NORM": "0"}),
):
hint = flashinfer_rmsnorm_diagnostic_hint()
self.assertIn("backend=CuTe DSL", hint)
self.assertIn("FLASHINFER_USE_CUDA_NORM=0", hint)
for package, version in versions.items():
self.assertIn(f"{package}={version}", hint)
# ---------------------------------------------------------------------------
# Mount lifecycle on synthetic sites (CUDA)
# ---------------------------------------------------------------------------
LN_DIM = 3072 # FLUX.1 hidden size
@requires_cuda
@pytest.mark.parametrize("seq_len", [4096, 512])
def test_fused_ln_modulate_matches_reference(seq_len):
torch.cuda.manual_seed(0)
x = torch.randn((1, seq_len, LN_DIM), device="cuda", dtype=torch.bfloat16)
scale = torch.randn((1, LN_DIM), device="cuda", dtype=torch.bfloat16)
shift = torch.randn_like(scale)
assert can_use_ln_modulate(x, scale, shift)
norm = nn.LayerNorm(LN_DIM, eps=1e-6, elementwise_affine=False).cuda()
ref = norm(x) * (1 + scale[:, None]) + shift[:, None]
# Contract: bf16 rounding-order-level difference only, not bit-exact --
# which is exactly why this fusion is quality-gated.
torch.testing.assert_close(
fused_ln_modulate(x, scale, shift, eps=1e-6), ref, atol=0.0625, rtol=0.05
)
@requires_cuda
def test_fused_ln_modulate_guards_and_mount_protocol():
x = torch.randn((2, 64, LN_DIM), device="cuda", dtype=torch.bfloat16)
row = torch.randn((2, LN_DIM), device="cuda", dtype=torch.bfloat16)
assert not can_use_ln_modulate(x, row, row) # folded affine needs B == 1
root = nn.Module()
root.child = nn.Module()
FUSED_LN_MODULATE.mark(root.child)
assert not FUSED_LN_MODULATE.is_enabled(root.child)
assert FUSED_LN_MODULATE.mount(root)
assert FUSED_LN_MODULATE.is_enabled(root.child)
FUSED_LN_MODULATE.unmount(root)
assert not FUSED_LN_MODULATE.is_enabled(root.child)
assert not FUSED_LN_MODULATE.mount(nn.Module()) # no marked sites
@requires_cuda
@torch.no_grad()
def test_mounted_ln_modulate_site_compiles_fullgraph():
class Site(nn.Module):
def __init__(self):
super().__init__()
FUSED_LN_MODULATE.mark(self)
def forward(self, x, scale, shift):
if FUSED_LN_MODULATE.is_enabled(self) and can_use_ln_modulate(
x, scale, shift
):
return fused_ln_modulate(x, scale, shift, eps=1e-6)
return (
F.layer_norm(x, (x.shape[-1],), eps=1e-6) * (1 + scale[:, None])
+ shift[:, None]
)
site = Site()
assert FUSED_LN_MODULATE.mount(site)
x = torch.randn(1, 64, 128, device="cuda", dtype=torch.bfloat16)
scale = torch.randn(1, 128, device="cuda", dtype=torch.bfloat16)
shift = torch.randn_like(scale)
expected = site(x, scale, shift)
torch.testing.assert_close(
torch.compile(site, fullgraph=True)(x, scale, shift),
expected,
atol=0.0625,
rtol=0.05,
)
GATE_RMSNORM_DIM, GATE_RMSNORM_EPS = 4608, 1e-5 # Ideogram 4 hidden / norm_eps
class _GateRMSNormSite(nn.Module):
def __init__(self, dtype=torch.bfloat16):
super().__init__()
self.norm = nn.RMSNorm(
GATE_RMSNORM_DIM, eps=GATE_RMSNORM_EPS, device="cuda", dtype=dtype
)
gate_rmsnorm.FUSED_GATE_RMSNORM.mark(self, ("norm",))
@requires_cuda
def test_fused_gate_rmsnorm_matches_ideogram_reference():
torch.manual_seed(0)
site = _GateRMSNormSite()
w = site.norm.weight.data
dim = GATE_RMSNORM_DIM
x = torch.randn(1, 64, dim, device="cuda", dtype=torch.bfloat16)
residual = torch.randn_like(x)
# adaln-style strided chunks, as produced by Ideogram's modulation .chunk()
mods = torch.randn(1, 1, 2 * dim, device="cuda", dtype=torch.bfloat16)
scale, gate = mods.chunk(2, dim=-1)
assert gate_rmsnorm.FUSED_GATE_RMSNORM.mount(site)
got_scale = gate_rmsnorm.fused_rmsnorm_scale(x, w, 1.0 + scale, GATE_RMSNORM_EPS)
got_gate = gate_rmsnorm.fused_rmsnorm_tanh_residual(
x, gate, residual, w, GATE_RMSNORM_EPS
)
norm = F.rms_norm(x, (dim,), w, GATE_RMSNORM_EPS)
# The fused path uses bf16-native norm statistics: close, not bit-exact.
torch.testing.assert_close(got_scale, norm * (1.0 + scale), atol=8e-2, rtol=4e-2)
torch.testing.assert_close(
got_gate, residual + torch.tanh(gate) * norm, atol=8e-2, rtol=4e-2
)
@requires_cuda
def test_fused_gate_rmsnorm_mount_is_all_or_nothing():
good, bad = _GateRMSNormSite(), _GateRMSNormSite(torch.float32)
# One fp32 norm anywhere in the tree keeps *every* site on the reference.
assert not gate_rmsnorm.FUSED_GATE_RMSNORM.mount(nn.ModuleList([good, bad]))
assert not gate_rmsnorm.FUSED_GATE_RMSNORM.is_enabled(good)
assert gate_rmsnorm.FUSED_GATE_RMSNORM.mount(good)
gate_rmsnorm.FUSED_GATE_RMSNORM.unmount(good)
assert not gate_rmsnorm.FUSED_GATE_RMSNORM.is_enabled(good)
class _GeluSite(nn.Module):
def __init__(self, dtype=torch.bfloat16, bias=True):
super().__init__()
self.proj = nn.Linear(64, 256, bias=bias, device="cuda", dtype=dtype)
linear_gelu.FUSED_LINEAR_GELU.mark(self, "proj")
def forward(self, x):
if linear_gelu.FUSED_LINEAR_GELU.is_enabled(
self
) and linear_gelu.can_use_linear_gelu(self.proj, x):
return linear_gelu.fused_linear_gelu_tanh(
x, self.proj.weight, self.proj.bias
)
return F.gelu(self.proj(x), approximate="tanh")
@requires_cuda
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_linear_gelu_matches_reference(dtype):
torch.manual_seed(0)
site = _GeluSite(dtype)
x = torch.randn(512, 64, device="cuda", dtype=dtype)
ref = site(x) # unmounted: the reference chain
assert linear_gelu.FUSED_LINEAR_GELU.mount(site)
atol = 2e-2 if dtype == torch.bfloat16 else 4e-3
torch.testing.assert_close(site(x), ref, atol=atol, rtol=2e-2)
@requires_cuda
def test_fused_linear_gelu_guards_and_lossless_path():
torch.manual_seed(0)
good, bad = _GeluSite(), _GeluSite(torch.float32)
assert not linear_gelu.FUSED_LINEAR_GELU.mount(nn.ModuleList([good, bad]))
assert not linear_gelu.FUSED_LINEAR_GELU.is_enabled(good)
x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16)
ref = good(x)
assert linear_gelu.FUSED_LINEAR_GELU.mount(good)
linear_gelu.FUSED_LINEAR_GELU.unmount(good)
# Unmounting must restore the reference path bit-for-bit.
assert torch.equal(good(x), ref)
no_bias = nn.Linear(8, 8, bias=False, device="cuda", dtype=torch.bfloat16)
assert not linear_gelu.can_use_linear_gelu_static(no_bias)
assert not linear_gelu.can_use_linear_gelu(good.proj, x.float())
@requires_cuda
def test_linear_gelu_verdict_is_reused_until_remount(monkeypatch):
if torch.cuda.get_device_capability() >= (12, 0):
pytest.skip(
"SM120 keeps these sites on the eager path before reading the linear"
)
site = _GeluSite()
x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16)
assert linear_gelu.FUSED_LINEAR_GELU.mount(site)
reads = []
static = linear_gelu.can_use_linear_gelu_static
monkeypatch.setattr(
linear_gelu,
"can_use_linear_gelu_static",
lambda linear: reads.append(linear) or static(linear),
)
for _ in range(3):
assert linear_gelu.can_use_linear_gelu(site.proj, x)
assert reads == [site.proj]
linear_gelu.FUSED_LINEAR_GELU.unmount(site)
assert linear_gelu.FUSED_LINEAR_GELU.mount(site)
assert linear_gelu.can_use_linear_gelu(site.proj, x)
assert reads == [site.proj, site.proj]
@requires_cuda
@torch.no_grad()
def test_mounted_gelu_site_compiles_fullgraph():
site = _GeluSite()
x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16)
assert linear_gelu.FUSED_LINEAR_GELU.mount(site)
expected = site(x)
torch.testing.assert_close(
torch.compile(site, fullgraph=True)(x), expected, atol=2e-2, rtol=2e-2
)
@requires_cuda
@torch.no_grad()
def test_sana_video_linear_attention_quality_path_and_guards():
torch.manual_seed(0)
site = nn.Module()
sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.mark(site)
shape = (1, 4, 16, 128)
query = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
key = torch.randn_like(query)
value = torch.randn_like(query)
normalizer = torch.randn(
shape[0], shape[1], 1, shape[-1], device="cuda", dtype=torch.bfloat16
)
assert (
sana_video_linear_attention.try_sana_video_linear_attention(
site, query, key, value, normalizer
)
is None
)
assert sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.mount(site)
output = sana_video_linear_attention.try_sana_video_linear_attention(
site, query, key, value, normalizer
)
reference = ((value.float() @ key.float().transpose(-1, -2)) @ query.float()) * (
normalizer
)
torch.testing.assert_close(output, reference, atol=1e-2, rtol=1e-2)
assert (
sana_video_linear_attention.try_sana_video_linear_attention(
site,
query.expand(2, -1, -1, -1),
key.expand(2, -1, -1, -1),
value.expand(2, -1, -1, -1),
normalizer.expand(2, -1, -1, -1),
)
is None
)
sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.unmount(site)
assert not sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.is_enabled(site)
@requires_cuda
@torch.no_grad()
def test_lingbot_video_rmsnorm_quality_path_and_guards():
torch.manual_seed(1)
site = nn.Module()
site.weight = nn.Parameter(torch.randn(2048, device="cuda", dtype=torch.float32))
lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mark(site)
hidden_states = torch.randn(1, 128, 2048, device="cuda", dtype=torch.bfloat16)
assert (
lingbot_video_rmsnorm.try_lingbot_video_rmsnorm(
site, hidden_states, site.weight, 1e-6
)
is None
)
assert lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mount(site)
output = lingbot_video_rmsnorm.try_lingbot_video_rmsnorm(
site, hidden_states, site.weight, 1e-6
)
states_fp32 = hidden_states.float()
variance = states_fp32.pow(2).mean(-1, keepdim=True)
normalized = states_fp32 * torch.rsqrt(variance + 1e-6)
reference = (site.weight * normalized).bfloat16()
torch.testing.assert_close(output, reference, atol=2e-2, rtol=2e-2)
bf16_site = nn.Module()
bf16_site.weight = nn.Parameter(
torch.randn(2048, device="cuda", dtype=torch.bfloat16)
)
lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mark(bf16_site)
assert lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mount(bf16_site)
bf16_output = lingbot_video_rmsnorm.try_lingbot_video_rmsnorm(
bf16_site, hidden_states, bf16_site.weight, 1e-6
)
bf16_reference = (bf16_site.weight * normalized).bfloat16()
torch.testing.assert_close(bf16_output, bf16_reference, atol=3e-2, rtol=3e-2)
assert (
lingbot_video_rmsnorm.try_lingbot_video_rmsnorm(
site, hidden_states.float(), site.weight, 1e-6
)
is None
)
lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.unmount(site)
assert not lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.is_enabled(site)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))