185 lines
5.3 KiB
Python
185 lines
5.3 KiB
Python
import sys
|
|
|
|
import pytest
|
|
import sgl_kernel # noqa: F401
|
|
import torch
|
|
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.cpu_test_utils import precision
|
|
|
|
register_cpu_ci(est_time=5, suite="stage-a-test-cpu-intel")
|
|
register_cpu_ci(est_time=10, suite="base-b-test-cpu-arm64")
|
|
|
|
torch.manual_seed(1234)
|
|
|
|
eps = 1e-6
|
|
|
|
DTYPE_PAIRS = [
|
|
(torch.bfloat16, torch.bfloat16),
|
|
(torch.bfloat16, torch.float32),
|
|
(torch.float16, torch.float16),
|
|
(torch.float16, torch.float32),
|
|
]
|
|
|
|
|
|
class TestDiffusionNorm:
|
|
def rmsnorm_ref(
|
|
self,
|
|
x: torch.Tensor,
|
|
weight: torch.Tensor | None,
|
|
eps: float,
|
|
) -> torch.Tensor:
|
|
x_fp32 = x.float()
|
|
variance = x_fp32.square().mean(dim=-1, keepdim=True)
|
|
out = x_fp32 * torch.rsqrt(variance + eps)
|
|
|
|
if weight is not None:
|
|
out = out * weight.float()
|
|
|
|
return out
|
|
|
|
@pytest.mark.parametrize("input_dtype,param_dtype", DTYPE_PAIRS)
|
|
@pytest.mark.parametrize("broadcast_c", [False, True])
|
|
def test_fused_scale_shift(
|
|
self,
|
|
input_dtype,
|
|
param_dtype,
|
|
broadcast_c,
|
|
):
|
|
B, S, D = 2, 4, 67
|
|
x = torch.randn(B, S, D, dtype=input_dtype)
|
|
|
|
if broadcast_c:
|
|
# hidden dimension broadcast -> stride_c == 0
|
|
scale = torch.randn(B, 1, 1, dtype=param_dtype)
|
|
shift = torch.randn(B, S, 1, dtype=param_dtype)
|
|
else:
|
|
# normal vector load -> stride_c == 1
|
|
scale = torch.randn(B, 1, D, dtype=param_dtype)
|
|
shift = torch.randn(B, S, D, dtype=param_dtype)
|
|
|
|
scale_expanded = scale.expand_as(x)
|
|
shift_expanded = shift.expand_as(x)
|
|
|
|
if broadcast_c:
|
|
assert scale_expanded.stride(2) == 0
|
|
assert shift_expanded.stride(2) == 0
|
|
else:
|
|
assert scale_expanded.stride(2) == 1
|
|
assert shift_expanded.stride(2) == 1
|
|
|
|
out = torch.ops.sgl_kernel.fused_scale_shift_cpu(
|
|
x,
|
|
scale_expanded,
|
|
shift_expanded,
|
|
1.0,
|
|
)
|
|
|
|
ref = (x.float() * (1.0 + scale.float()) + shift.float()).to(input_dtype)
|
|
|
|
torch.testing.assert_close(
|
|
out,
|
|
ref,
|
|
atol=precision[input_dtype],
|
|
rtol=precision[input_dtype],
|
|
)
|
|
|
|
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
|
@pytest.mark.parametrize(
|
|
"gate_type,norm_dtype,param_type,norm_type",
|
|
[
|
|
("input", None, "input", "rms"),
|
|
("fp32", torch.float32, "input", "layer"),
|
|
(None, None, "fp32", "layer"),
|
|
],
|
|
)
|
|
def test_fused_scale_residual_norm_scale_shift(
|
|
self,
|
|
input_dtype,
|
|
gate_type,
|
|
norm_dtype,
|
|
param_type,
|
|
norm_type,
|
|
):
|
|
B, S, D = 2, 4, 67
|
|
|
|
x = torch.randn(B, S, D, dtype=input_dtype)
|
|
residual = torch.randn(B, S, D, dtype=input_dtype)
|
|
|
|
gate_dtype = (
|
|
input_dtype
|
|
if gate_type == "input"
|
|
else torch.float32
|
|
if gate_type == "fp32"
|
|
else None
|
|
)
|
|
param_dtype = input_dtype if param_type == "input" else torch.float32
|
|
|
|
gate = torch.randn(D, dtype=gate_dtype) if gate_dtype is not None else None
|
|
weight = torch.randn(D, dtype=norm_dtype) if norm_dtype is not None else None
|
|
bias = (
|
|
torch.randn(D, dtype=norm_dtype)
|
|
if norm_dtype is not None and norm_type == "layer"
|
|
else None
|
|
)
|
|
|
|
scale = torch.randn(B, 1, D, dtype=param_dtype)
|
|
shift = torch.randn(B, S, D, dtype=param_dtype)
|
|
|
|
scale_expanded = scale.expand_as(x)
|
|
shift_expanded = shift.expand_as(x)
|
|
gate_expanded = gate.view(1, 1, D).expand_as(x) if gate is not None else None
|
|
|
|
out, residual_out = (
|
|
torch.ops.sgl_kernel.fused_scale_residual_norm_scale_shift_cpu(
|
|
residual,
|
|
x,
|
|
gate_expanded,
|
|
weight,
|
|
bias,
|
|
scale_expanded,
|
|
shift_expanded,
|
|
norm_type,
|
|
eps=eps,
|
|
)
|
|
)
|
|
|
|
if gate is None:
|
|
residual_fp32 = residual.float() + x.float()
|
|
else:
|
|
residual_fp32 = residual.float() + x.float() * gate.float()
|
|
|
|
ref_residual = residual_fp32.to(input_dtype)
|
|
norm_input = ref_residual.float()
|
|
|
|
if norm_type == "rms":
|
|
normalized = self.rmsnorm_ref(norm_input, weight, eps)
|
|
else:
|
|
normalized = torch.nn.functional.layer_norm(
|
|
norm_input,
|
|
(D,),
|
|
weight.float() if weight is not None else None,
|
|
bias.float() if bias is not None else None,
|
|
eps,
|
|
)
|
|
|
|
# Match CUDA activation boundary after norm.
|
|
normalized = normalized.to(input_dtype).float()
|
|
|
|
ref_out = (normalized * (1.0 + scale.float()) + shift.float()).to(input_dtype)
|
|
|
|
torch.testing.assert_close(
|
|
residual_out,
|
|
ref_residual,
|
|
atol=precision[input_dtype],
|
|
rtol=precision[input_dtype],
|
|
)
|
|
|
|
torch.testing.assert_close(
|
|
out, ref_out, atol=precision[input_dtype], rtol=precision[input_dtype]
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__]))
|