162 lines
4.6 KiB
Python
162 lines
4.6 KiB
Python
import pytest
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from sglang.kernels.ops.elementwise.hc_combine import hc_combine
|
|
from sglang.test.ci.ci_register import register_cuda_ci
|
|
|
|
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
|
|
|
HC_COUNT = 4
|
|
HIDDEN_SIZE = 2560
|
|
|
|
|
|
def _reference_hc_combine(
|
|
block_output: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
normed_residual: torch.Tensor,
|
|
inject_weight: torch.Tensor,
|
|
hc: int,
|
|
hs: int,
|
|
compute_dtype: torch.dtype = torch.float32,
|
|
) -> torch.Tensor:
|
|
"""Eager reference mirroring ``GatedResidual._combine_compute``;
|
|
``compute_dtype=torch.float64`` is the near-exact reference."""
|
|
R = residual.to(compute_dtype).unflatten(-1, (hc, hs))
|
|
gates = 2 * torch.sigmoid(
|
|
F.linear(normed_residual.to(compute_dtype), inject_weight.to(compute_dtype))
|
|
/ hc
|
|
)
|
|
injection = block_output.to(compute_dtype).unsqueeze(-2) * gates.unsqueeze(-1)
|
|
return (R + injection).flatten(-2)
|
|
|
|
|
|
def _make_inputs(
|
|
num_tokens: int, dtype: torch.dtype, hc: int = HC_COUNT, hs: int = HIDDEN_SIZE
|
|
):
|
|
torch.manual_seed(0)
|
|
block_output = torch.randn(num_tokens, hs, dtype=dtype, device="cuda")
|
|
residual = torch.randn(num_tokens, hc * hs, dtype=dtype, device="cuda")
|
|
normed_residual = torch.randn(num_tokens, hc * hs, dtype=dtype, device="cuda")
|
|
inject_weight = torch.randn(hc, hc * hs, dtype=dtype, device="cuda") * 0.02
|
|
return block_output, residual, normed_residual, inject_weight
|
|
|
|
|
|
# Worst case over M in {1, 7, 128, 8192} against the fp64 reference on B300 (sm103):
|
|
# bf16 max rel err 7.8e-3 (1 ulp at a binade edge), fp16 below 1e-3 (1 ulp = 9.8e-4);
|
|
# the residual is fp32 reordering flipping the final rounding.
|
|
_TOLERANCES = {
|
|
torch.bfloat16: dict(rtol=1e-2, atol=5e-3),
|
|
torch.float16: dict(rtol=1e-3, atol=1e-3),
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
|
@pytest.mark.parametrize("num_tokens", [1, 7, 128, 8192])
|
|
def test_hc_combine_correctness(dtype, num_tokens):
|
|
block_output, residual, normed_residual, inject_weight = _make_inputs(
|
|
num_tokens, dtype
|
|
)
|
|
|
|
out = hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
)
|
|
expected = _reference_hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
compute_dtype=torch.float64,
|
|
).to(dtype)
|
|
|
|
torch.testing.assert_close(out, expected, **_TOLERANCES[dtype])
|
|
|
|
|
|
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
|
def test_hc_combine_out_param(dtype):
|
|
block_output, residual, normed_residual, inject_weight = _make_inputs(64, dtype)
|
|
out = torch.empty_like(residual)
|
|
|
|
result = hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
out=out,
|
|
)
|
|
expected = _reference_hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
compute_dtype=torch.float64,
|
|
).to(dtype)
|
|
|
|
assert result.data_ptr() == out.data_ptr()
|
|
torch.testing.assert_close(result, expected, **_TOLERANCES[dtype])
|
|
|
|
|
|
def test_hc_combine_3d_input():
|
|
dtype = torch.bfloat16
|
|
block_output, residual, normed_residual, inject_weight = _make_inputs(32, dtype)
|
|
block_output = block_output.reshape(4, 8, HIDDEN_SIZE)
|
|
residual = residual.reshape(4, 8, HC_COUNT * HIDDEN_SIZE)
|
|
normed_residual = normed_residual.reshape(4, 8, HC_COUNT * HIDDEN_SIZE)
|
|
|
|
out = hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
)
|
|
expected = _reference_hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
HC_COUNT,
|
|
HIDDEN_SIZE,
|
|
compute_dtype=torch.float64,
|
|
).to(dtype)
|
|
|
|
assert out.shape == residual.shape
|
|
torch.testing.assert_close(out, expected, **_TOLERANCES[dtype])
|
|
|
|
|
|
def test_hc_combine_bad_hidden_size():
|
|
dtype = torch.bfloat16
|
|
block_output, residual, normed_residual, inject_weight = _make_inputs(
|
|
4,
|
|
dtype,
|
|
hc=4,
|
|
hs=1000, # 4 * 1000 = 4000, not a multiple of 2048
|
|
)
|
|
with pytest.raises(RuntimeError, match="2048"):
|
|
hc_combine(
|
|
block_output,
|
|
residual,
|
|
normed_residual,
|
|
inject_weight,
|
|
4,
|
|
1000,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
sys.exit(pytest.main([__file__, "-v", "-s"]))
|