Files
sglang/test/registered/kernels/ops/diffusion/test_complex_rope.py
T

374 lines
12 KiB
Python

# SPDX-License-Identifier: Apache-2.0
import sys
import pytest
import torch
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_complex_rope,
can_use_qknorm_complex_rope_cuda,
can_use_qknorm_complex_rope_pack,
fused_complex_rope,
qknorm_complex_rope_cuda,
qknorm_complex_rope_pack_,
)
from sglang.multimodal_gen.runtime.models.dits import qwen_image21
from sglang.srt.layers.layernorm import RMSNorm
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or torch.version.hip is not None,
reason="NVIDIA CUDA required",
)
def reference(x, rope):
z = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
return torch.view_as_real(z * rope[None, :, None]).flatten(-2).to(x.dtype)
def inputs(shape, dtype):
torch.manual_seed(42)
x = torch.randn(shape, device="cuda", dtype=dtype)
# a contiguous slice retains the nonzero cache offset used by SP ranks
angles = torch.randn(shape[1] + 5, shape[-1] // 2, device="cuda") * 20
rope = torch.polar(torch.ones_like(angles), angles)[5:]
return x, rope
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize(
"shape", [(1, 1, 1, 32), (2, 17, 3, 64), (1, 257, 16, 128), (1, 4096, 32, 128)]
)
def test_complex_rope_matches_complex_multiplication(dtype, shape):
x, rope = inputs(shape, dtype)
assert can_use_fused_complex_rope(x, rope)
actual = fused_complex_rope(x, rope)
torch.testing.assert_close(actual, reference(x, rope), atol=0, rtol=0)
def test_complex_rope_layout_guards():
x, rope = inputs((2, 17, 3, 64), torch.bfloat16)
assert not can_use_fused_complex_rope(x.cpu(), rope.cpu())
assert not can_use_fused_complex_rope(x.double(), rope)
assert not can_use_fused_complex_rope(x, rope.to(torch.complex128))
assert not can_use_fused_complex_rope(x[:, ::2], rope[::2])
assert not can_use_fused_complex_rope(x, rope[:-1])
assert not can_use_fused_complex_rope(x[:, :0], rope[:0])
def test_complex_rope_compile_and_graph_replay():
x, rope = inputs((1, 257, 8, 128), torch.bfloat16)
compiled = torch.compile(fused_complex_rope, fullgraph=True)
torch.testing.assert_close(compiled(x, rope), reference(x, rope), atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fused_complex_rope(x, rope)
x.normal_()
graph.replay()
torch.testing.assert_close(out, reference(x, rope), atol=0, rtol=0)
def test_qwen21_rope_first_sight_verification(monkeypatch):
x, rope = inputs((1, 257, 8, 128), torch.bfloat16)
gate = BitExactFusionGate("test complex RoPE")
monkeypatch.setattr(qwen_image21, "_ROPE_FUSION", gate)
torch.testing.assert_close(
qwen_image21.apply_rope(x, rope), reference(x, rope), atol=0, rtol=0
)
assert gate.verified and not gate.disabled
gate = BitExactFusionGate("test mismatched RoPE")
monkeypatch.setattr(qwen_image21, "_ROPE_FUSION", gate)
monkeypatch.setattr(
qwen_image21, "fused_complex_rope", lambda x, rope: torch.zeros_like(x)
)
torch.testing.assert_close(
qwen_image21.apply_rope(x, rope), reference(x, rope), atol=0, rtol=0
)
assert gate.disabled and not gate.verified
# -------------------------------------------------------------------------
# Qwen-Image 2.1 -- CUDA Q/K RMSNorm + complex RoPE, and the packed K/V variant
# -------------------------------------------------------------------------
EPS = 1e-6
HEAD_DIM = 128
def make_norm():
norm = RMSNorm(HEAD_DIM, EPS, cast_x_before_out_mul=True, force_native=True).to(
device="cuda", dtype=torch.bfloat16
)
norm.weight.normal_()
return norm
def make_rope(seq, scale=20.0):
angles = torch.randn(seq, HEAD_DIM // 2, device="cuda") * scale
return torch.polar(torch.ones_like(angles), angles)
def eager_norm_rope(x, norm, rope):
normed = norm(x)
z = torch.view_as_complex(normed.float().reshape(*normed.shape[:-1], -1, 2))
return torch.view_as_real(z * rope[None, :, None]).flatten(-2).to(x.dtype)
def assert_bits_equal(actual, expected):
assert actual.dtype is expected.dtype and actual.shape == expected.shape
assert torch.equal(actual.view(torch.int16), expected.view(torch.int16))
@torch.no_grad()
@pytest.mark.parametrize(
"shape", [(1, 4096, 32, 128), (1, 18, 32, 128), (2, 17, 4, 128), (1, 1, 1, 128)]
)
@pytest.mark.parametrize("amplitude", [1e-3, 1.0, 40.0])
def test_norm_rope_matches_eager(shape, amplitude):
torch.manual_seed(0)
norm = make_norm()
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16) * amplitude
rope = make_rope(shape[1])
assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
out = qknorm_complex_rope_cuda(x, norm.weight, rope, EPS)
assert_bits_equal(out, eager_norm_rope(x, norm, rope))
@torch.no_grad()
def test_norm_rope_is_sensitive_to_fma_orientation():
# The two complex-multiply FMA orientations differ on a large random sample;
# the equality test above therefore also pins the orientation probe.
torch.manual_seed(1)
x = torch.randn(1, 4096, 32, 128, device="cuda", dtype=torch.bfloat16)
rope = make_rope(4096)
normed = x # rotation only: pretend the norm is the identity for this check
z = torch.view_as_complex(normed.float().reshape(*normed.shape[:-1], -1, 2))
re, im = z.real, z.imag
c, s = rope.real[None, :, None], rope.imag[None, :, None]
imag_a = torch.addcmul(im * c, re, s).to(torch.bfloat16)
imag_b = torch.addcmul(re * s, im, c).to(torch.bfloat16)
assert not torch.equal(imag_a, imag_b)
def _pack_inputs(batch, seq, prefix, heads):
torch.manual_seed(2)
q = torch.randn(batch, seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
k = torch.randn_like(q)
v = torch.randn_like(q)
kp = torch.randn(
batch, prefix, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16
)
vp = torch.randn_like(kp)
rope = make_rope(seq)
return q, k, v, kp, vp, rope
@torch.no_grad()
@pytest.mark.parametrize("batch,seq,prefix,heads", [(1, 4096, 18, 32), (2, 17, 5, 4)])
@pytest.mark.parametrize("in_place", [False, True])
def test_pack_matches_eager(batch, seq, prefix, heads, in_place):
q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads)
norm_q, norm_k = make_norm(), make_norm()
expected_q = eager_norm_rope(q, norm_q, rope)
expected_k = torch.cat([kp, eager_norm_rope(k, norm_k, rope)], dim=1)
expected_v = torch.cat([vp, v], dim=1)
k_out = torch.empty(
batch, prefix + seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16
)
v_out = torch.empty_like(k_out)
if in_place:
# the projection wrote raw K and V into the token rows already
k_out[:, prefix:].copy_(k)
v_out[:, prefix:].copy_(v)
k_src, v_src = None, None
else:
k_src, v_src = k, v
q_work = q.clone()
assert can_use_qknorm_complex_rope_pack(
q_work, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, k_src, v_src
)
qknorm_complex_rope_pack_(
q_work,
k_out,
v_out,
norm_q.weight,
norm_k.weight,
rope,
kp,
vp,
k_src,
v_src,
EPS,
)
assert_bits_equal(q_work, expected_q)
assert_bits_equal(k_out, expected_k)
assert_bits_equal(v_out, expected_v)
@torch.no_grad()
def test_pack_accepts_padded_token_stride():
# Column views of a wider [B, P+S, 3C] buffer: heads contiguous, token stride 3C.
batch, seq, prefix, heads = 1, 33, 4, 8
q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads)
norm_q, norm_k = make_norm(), make_norm()
wide = torch.empty(
batch, prefix + seq, 3 * heads * HEAD_DIM, device="cuda", dtype=torch.bfloat16
)
hd = heads * HEAD_DIM
q_view = wide[:, prefix:, :hd].view(batch, seq, heads, HEAD_DIM)
k_out = wide[:, :, hd : 2 * hd].view(batch, prefix + seq, heads, HEAD_DIM)
v_out = wide[:, :, 2 * hd :].view(batch, prefix + seq, heads, HEAD_DIM)
q_view.copy_(q)
k_out[:, prefix:].copy_(k)
v_out[:, prefix:].copy_(v)
assert can_use_qknorm_complex_rope_pack(
q_view, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, None, None
)
qknorm_complex_rope_pack_(
q_view,
k_out,
v_out,
norm_q.weight,
norm_k.weight,
rope,
kp,
vp,
None,
None,
EPS,
)
assert_bits_equal(q_view, eager_norm_rope(q, norm_q, rope))
assert_bits_equal(k_out, torch.cat([kp, eager_norm_rope(k, norm_k, rope)], dim=1))
assert_bits_equal(v_out, torch.cat([vp, v], dim=1))
@torch.no_grad()
def test_predicates_reject_unsupported_layouts():
norm = make_norm()
x = torch.randn(1, 17, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
rope = make_rope(17)
assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
assert not can_use_qknorm_complex_rope_cuda(x.cpu(), norm.weight.cpu(), rope.cpu())
assert not can_use_qknorm_complex_rope_cuda(x.half(), norm.weight.half(), rope)
assert not can_use_qknorm_complex_rope_cuda(
x[..., :64], norm.weight[:64], rope[:, :32]
)
assert not can_use_qknorm_complex_rope_cuda(x.transpose(1, 2), norm.weight, rope)
assert not can_use_qknorm_complex_rope_cuda(x, norm.weight, rope[:-1])
assert not can_use_qknorm_complex_rope_cuda(
x, norm.weight, rope.to(torch.complex128)
)
assert not can_use_qknorm_complex_rope_cuda(x, norm.weight.float(), rope)
k_out = torch.empty(1, 17 + 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
kp = torch.randn(1, 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
assert can_use_qknorm_complex_rope_pack(
x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, None, None
)
# wrong prefix + seq length, batch mismatch, non-contiguous prefix, short K source
assert not can_use_qknorm_complex_rope_pack(
x,
k_out[:, :-1],
k_out.clone(),
norm.weight,
norm.weight,
rope,
kp,
kp,
None,
None,
)
assert not can_use_qknorm_complex_rope_pack(
x,
k_out,
k_out.clone(),
norm.weight,
norm.weight,
rope,
kp.expand(2, -1, -1, -1),
kp,
None,
None,
)
assert not can_use_qknorm_complex_rope_pack(
x,
k_out,
k_out.clone(),
norm.weight,
norm.weight,
rope,
kp.transpose(1, 2),
kp,
None,
None,
)
assert not can_use_qknorm_complex_rope_pack(
x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, x[:, :-1], None
)
@torch.no_grad()
def test_pack_graph_capture_and_replay():
batch, seq, prefix, heads = 1, 65, 3, 8
q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads)
norm_q, norm_k = make_norm(), make_norm()
q_work = q.clone()
k_out = torch.empty(
batch, prefix + seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16
)
v_out = torch.empty_like(k_out)
k_out[:, prefix:].copy_(k)
v_out[:, prefix:].copy_(v)
# warm the JIT module outside the capture
qknorm_complex_rope_pack_(
q_work.clone(),
k_out.clone(),
v_out.clone(),
norm_q.weight,
norm_k.weight,
rope,
kp,
vp,
None,
None,
EPS,
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
qknorm_complex_rope_pack_(
q_work,
k_out,
v_out,
norm_q.weight,
norm_k.weight,
rope,
kp,
vp,
None,
None,
EPS,
)
# new inputs through the captured buffers
q2, k2, v2, kp2, vp2, _ = _pack_inputs(batch, seq, prefix, heads)
q_work.copy_(q2)
k_out[:, prefix:].copy_(k2)
v_out[:, prefix:].copy_(v2)
kp.copy_(kp2)
vp.copy_(vp2)
graph.replay()
torch.cuda.synchronize()
assert_bits_equal(q_work, eager_norm_rope(q2, norm_q, rope))
assert_bits_equal(k_out, torch.cat([kp2, eager_norm_rope(k2, norm_k, rope)], dim=1))
assert_bits_equal(v_out, torch.cat([vp2, v2], dim=1))
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))