[Kernel Slimming] Migrate NVFP4 kernels to JIT (#19437)
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.jit_kernel.nvfp4 import (
|
||||
cutlass_fp4_group_mm,
|
||||
scaled_fp4_experts_quant,
|
||||
scaled_fp4_quant,
|
||||
)
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
|
||||
def _nvfp4_supported() -> bool:
|
||||
return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (10, 0)
|
||||
|
||||
|
||||
def _round_up(x: int, y: int) -> int:
|
||||
return ((x + y - 1) // y) * y
|
||||
|
||||
|
||||
def _build_expert_offsets(
|
||||
m_per_expert: list[int], device: torch.device
|
||||
) -> torch.Tensor:
|
||||
offsets = [0]
|
||||
for m in m_per_expert:
|
||||
offsets.append(offsets[-1] + m)
|
||||
return torch.tensor(offsets, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
def _build_blockscale_offsets(
|
||||
m_per_expert: list[int], device: torch.device
|
||||
) -> torch.Tensor:
|
||||
offsets = [0]
|
||||
for m in m_per_expert:
|
||||
offsets.append(offsets[-1] + _round_up(m, 128))
|
||||
return torch.tensor(offsets, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_nvfp4_blockwise_moe_grouped_mm(dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
|
||||
num_experts = 4
|
||||
m_per_expert = [33, 17, 48, 29]
|
||||
n = 256
|
||||
k = 128
|
||||
|
||||
expert_offsets_full = _build_expert_offsets(m_per_expert, device)
|
||||
blockscale_offsets_full = _build_blockscale_offsets(m_per_expert, device)
|
||||
|
||||
total_m = int(expert_offsets_full[-1].item())
|
||||
a = torch.randn((total_m, k), device=device, dtype=dtype) * 0.1
|
||||
b = torch.randn((num_experts, n, k), device=device, dtype=dtype) * 0.1
|
||||
|
||||
a_global_scale = torch.empty((num_experts,), device=device, dtype=torch.float32)
|
||||
for i in range(num_experts):
|
||||
start = int(expert_offsets_full[i].item())
|
||||
end = int(expert_offsets_full[i + 1].item())
|
||||
amax = a[start:end].abs().max().to(torch.float32)
|
||||
a_global_scale[i] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / amax
|
||||
|
||||
b_global_scale = torch.empty((num_experts,), device=device, dtype=torch.float32)
|
||||
for i in range(num_experts):
|
||||
bmax = b[i].abs().max().to(torch.float32)
|
||||
b_global_scale[i] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / bmax
|
||||
|
||||
a_fp4, a_blockscale = scaled_fp4_experts_quant(
|
||||
a,
|
||||
a_global_scale,
|
||||
expert_offsets_full,
|
||||
blockscale_offsets_full,
|
||||
topk=1,
|
||||
)
|
||||
|
||||
b_fp4 = torch.empty((num_experts, n, k // 2), device=device, dtype=torch.uint8)
|
||||
b_blockscale = torch.empty(
|
||||
(num_experts, _round_up(n, 128), _round_up(k // 16, 4)),
|
||||
device=device,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
)
|
||||
for i in range(num_experts):
|
||||
b_fp4_i, b_scale_i = scaled_fp4_quant(b[i], b_global_scale[i])
|
||||
b_fp4[i].copy_(b_fp4_i)
|
||||
b_blockscale[i].copy_(b_scale_i)
|
||||
|
||||
alphas = (1.0 / (a_global_scale * b_global_scale)).to(torch.float32)
|
||||
|
||||
params = {
|
||||
"ab_strides": torch.full((num_experts,), k, dtype=torch.int64, device=device),
|
||||
"c_strides": torch.full((num_experts,), n, dtype=torch.int64, device=device),
|
||||
"problem_sizes": torch.tensor(
|
||||
[[m, n, k] for m in m_per_expert], dtype=torch.int32, device=device
|
||||
),
|
||||
"expert_offsets": expert_offsets_full[:-1].contiguous(),
|
||||
"blockscale_offsets": blockscale_offsets_full[:-1].contiguous(),
|
||||
"a_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"b_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"out_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"a_scales_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"b_scales_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"alpha_ptrs": torch.empty((num_experts,), dtype=torch.int64, device=device),
|
||||
"layout_sfa": torch.empty((num_experts, 5), dtype=torch.int64, device=device),
|
||||
"layout_sfb": torch.empty((num_experts, 5), dtype=torch.int64, device=device),
|
||||
}
|
||||
|
||||
out = cutlass_fp4_group_mm(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_blockscale,
|
||||
b_blockscale,
|
||||
alphas,
|
||||
dtype,
|
||||
params,
|
||||
)
|
||||
|
||||
ref = torch.empty((total_m, n), device=device, dtype=dtype)
|
||||
for i in range(num_experts):
|
||||
start = int(expert_offsets_full[i].item())
|
||||
end = int(expert_offsets_full[i + 1].item())
|
||||
ref[start:end] = torch.matmul(a[start:end], b[i].t())
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=1e-1, rtol=1e-1)
|
||||
@@ -0,0 +1,142 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.jit_kernel.nvfp4 import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
|
||||
|
||||
def _nvfp4_supported() -> bool:
|
||||
return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (10, 0)
|
||||
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
SHAPES = [
|
||||
(128, 128, 64),
|
||||
(128, 128, 128),
|
||||
(256, 128, 64),
|
||||
(128, 256, 128),
|
||||
(150, 128, 64),
|
||||
]
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
K_E2M1_TO_FLOAT = [
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
]
|
||||
|
||||
|
||||
def e2m1_to_fp32(int4_value: int) -> float:
|
||||
sign_bit = int4_value & 0x8
|
||||
int4_abs_value = int4_value & 0x7
|
||||
float_result = K_E2M1_TO_FLOAT[int4_abs_value]
|
||||
return -float_result if sign_bit else float_result
|
||||
|
||||
|
||||
def break_fp4_bytes(a: torch.Tensor) -> torch.Tensor:
|
||||
assert a.dtype == torch.uint8
|
||||
m, n = a.shape
|
||||
a = a.flatten()
|
||||
high_half_byte = (a & 0xF0) >> 4
|
||||
low_half_byte = a & 0x0F
|
||||
f_h = torch.tensor([e2m1_to_fp32(x) for x in high_half_byte], device=a.device)
|
||||
f_l = torch.tensor([e2m1_to_fp32(x) for x in low_half_byte], device=a.device)
|
||||
return torch.stack((f_l, f_h), dim=-1).reshape(m, n * 2)
|
||||
|
||||
|
||||
def convert_swizzled_to_linear(
|
||||
a_sf_swizzled: torch.Tensor, m: int, k: int, block_size: int
|
||||
) -> torch.Tensor:
|
||||
sf_m, sf_k = a_sf_swizzled.shape
|
||||
del sf_m, sf_k
|
||||
m_tiles = (m + 128 - 1) // 128
|
||||
f = block_size * 4
|
||||
k_tiles = (k + f - 1) // f
|
||||
tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4))
|
||||
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
|
||||
out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size)
|
||||
return out[0:m, 0 : k // block_size]
|
||||
|
||||
|
||||
def dequantize_to_dtype(
|
||||
tensor_fp4: torch.Tensor,
|
||||
tensor_sf: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
block_size: int = 16,
|
||||
) -> torch.Tensor:
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
m, packed_k = tensor_fp4.shape
|
||||
k = packed_k * 2
|
||||
tensor_f32 = break_fp4_bytes(tensor_fp4)
|
||||
tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size)
|
||||
tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
|
||||
tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size)
|
||||
tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale
|
||||
return (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k)
|
||||
|
||||
|
||||
def get_ref_results(
|
||||
a_fp4: torch.Tensor,
|
||||
b_fp4: torch.Tensor,
|
||||
a_sf: torch.Tensor,
|
||||
b_sf: torch.Tensor,
|
||||
a_global_scale: torch.Tensor,
|
||||
b_global_scale: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
a_in_dtype = dequantize_to_dtype(a_fp4, a_sf, a_global_scale, block_size=block_size)
|
||||
b_in_dtype = dequantize_to_dtype(b_fp4, b_sf, b_global_scale, block_size=block_size)
|
||||
return torch.matmul(a_in_dtype, b_in_dtype.t())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
def test_nvfp4_gemm(dtype: torch.dtype, shape: tuple[int, int, int]) -> None:
|
||||
m, n, packed_k = shape
|
||||
k = packed_k * 2
|
||||
block_size = 16
|
||||
|
||||
a_dtype = torch.randn((m, k), dtype=dtype, device="cuda")
|
||||
b_dtype = torch.randn((n, k), dtype=dtype, device="cuda")
|
||||
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
b_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
|
||||
alpha = 1.0 / (a_global_scale * b_global_scale)
|
||||
|
||||
a_fp4, a_scale_interleaved = scaled_fp4_quant(a_dtype, a_global_scale)
|
||||
b_fp4, b_scale_interleaved = scaled_fp4_quant(b_dtype, b_global_scale)
|
||||
|
||||
expected_out = get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_scale_interleaved,
|
||||
b_scale_interleaved,
|
||||
a_global_scale,
|
||||
b_global_scale,
|
||||
block_size,
|
||||
)
|
||||
|
||||
out = cutlass_scaled_fp4_mm(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_scale_interleaved,
|
||||
b_scale_interleaved,
|
||||
alpha,
|
||||
dtype,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
|
||||
@@ -0,0 +1,214 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.jit_kernel.nvfp4 import (
|
||||
scaled_fp4_grouped_quant,
|
||||
scaled_fp4_quant,
|
||||
silu_and_mul_scaled_fp4_grouped_quant,
|
||||
)
|
||||
|
||||
try:
|
||||
from sgl_kernel import silu_and_mul as _sgl_silu_and_mul
|
||||
except Exception:
|
||||
_sgl_silu_and_mul = None
|
||||
|
||||
|
||||
def _nvfp4_supported() -> bool:
|
||||
return torch.cuda.is_available() and torch.cuda.get_device_capability() >= (10, 0)
|
||||
|
||||
|
||||
def _silu_and_mul_reference(x: torch.Tensor) -> torch.Tensor:
|
||||
if _sgl_silu_and_mul is not None:
|
||||
return _sgl_silu_and_mul(x)
|
||||
k = x.shape[-1] // 2
|
||||
return torch.nn.functional.silu(x[:, :, :k]) * x[:, :, k:]
|
||||
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)]
|
||||
PAD_SHAPES = [
|
||||
(90, 64),
|
||||
(150, 64),
|
||||
(128, 48),
|
||||
(128, 80),
|
||||
]
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
E2M1_TO_FLOAT32 = [
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
]
|
||||
|
||||
|
||||
def cast_from_fp4(x: torch.Tensor, m: int, n: int) -> torch.Tensor:
|
||||
v_2nd = (x & 0xF).to(torch.long)
|
||||
v_1st = ((x >> 4) & 0xF).to(torch.long)
|
||||
c = torch.stack((v_2nd, v_1st), dim=-1).flatten()
|
||||
lut = torch.tensor(E2M1_TO_FLOAT32, device=x.device, dtype=torch.float32)
|
||||
return lut[c].reshape(m, n)
|
||||
|
||||
|
||||
def cast_to_fp4(x: torch.Tensor) -> torch.Tensor:
|
||||
sign = torch.sign(x)
|
||||
x = torch.abs(x)
|
||||
x[(x >= 0.0) & (x <= 0.25)] = 0.0
|
||||
x[(x > 0.25) & (x < 0.75)] = 0.5
|
||||
x[(x >= 0.75) & (x <= 1.25)] = 1.0
|
||||
x[(x > 1.25) & (x < 1.75)] = 1.5
|
||||
x[(x >= 1.75) & (x <= 2.5)] = 2.0
|
||||
x[(x > 2.5) & (x < 3.5)] = 3.0
|
||||
x[(x >= 3.5) & (x <= 5.0)] = 4.0
|
||||
x[x > 5.0] = 6.0
|
||||
return x * sign
|
||||
|
||||
|
||||
def get_reciprocal(x):
|
||||
if isinstance(x, torch.Tensor):
|
||||
return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
|
||||
return 0.0 if x == 0 else 1.0 / x
|
||||
|
||||
|
||||
def ref_nvfp4_quant(x: torch.Tensor, global_scale: torch.Tensor):
|
||||
assert global_scale.dtype == torch.float32
|
||||
assert x.ndim == 2
|
||||
m, n = x.shape
|
||||
x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE))
|
||||
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
|
||||
scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
|
||||
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
|
||||
|
||||
scaled_x = x.to(torch.float32) * output_scale
|
||||
clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
|
||||
return cast_to_fp4(clipped_x), scale.squeeze(-1)
|
||||
|
||||
|
||||
def recover_swizzled_scales(scale: torch.Tensor, m: int, n: int) -> torch.Tensor:
|
||||
rounded_m = ((m + 128 - 1) // 128) * 128
|
||||
scale_n = n // BLOCK_SIZE
|
||||
rounded_n = ((scale_n + 4 - 1) // 4) * 4
|
||||
tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
|
||||
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
|
||||
result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
|
||||
return result[:m, :scale_n]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
def test_quantize_to_fp4(dtype: torch.dtype, shape: tuple[int, int]) -> None:
|
||||
torch.manual_seed(42)
|
||||
m, n = shape
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype, device="cuda")
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = scaled_fp4_quant(x, global_scale)
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, n)
|
||||
out_ans = cast_from_fp4(out, m, n)
|
||||
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
torch.testing.assert_close(scale_ans, scale_ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("shape", PAD_SHAPES)
|
||||
def test_quantize_to_fp4_padded(shape: tuple[int, int]) -> None:
|
||||
torch.manual_seed(42)
|
||||
m, n = shape
|
||||
x = torch.randn((m, n), dtype=torch.float16, device="cuda")
|
||||
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = scaled_fp4_quant(x, global_scale)
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, n)
|
||||
out_ans = cast_from_fp4(out, m, n)
|
||||
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
torch.testing.assert_close(scale_ans, scale_ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(2, 128, 512), (2, 100, 128)])
|
||||
def test_quantize_to_fp4_grouped(shape: tuple[int, int, int]) -> None:
|
||||
torch.manual_seed(42)
|
||||
l, m, k = shape
|
||||
|
||||
x = torch.randn((l, m, k), dtype=torch.bfloat16, device="cuda")
|
||||
mask = torch.randint(1, max(2, m // 2), (l,), dtype=torch.int32, device="cuda")
|
||||
tensor_amax = x.abs().amax(dim=(1, 2)).to(torch.float32)
|
||||
x_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
|
||||
output, output_scales = scaled_fp4_grouped_quant(x, x_sf_global, mask)
|
||||
output = output.permute(2, 0, 1)
|
||||
padded_m = ((m + 128 - 1) // 128) * 128
|
||||
output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1)
|
||||
|
||||
for i in range(l):
|
||||
a_fp4, a_scale_interleaved = scaled_fp4_quant(x[i], x_sf_global[i])
|
||||
torch.testing.assert_close(a_fp4[: mask[i]], output[i][: mask[i]])
|
||||
scale_ref = recover_swizzled_scales(a_scale_interleaved, m, k)
|
||||
scale_ans = recover_swizzled_scales(output_scales[i], m, k)
|
||||
torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _nvfp4_supported(), reason="NVFP4 requires compute capability >= 10.0"
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(4, 96, 256), (8, 128, 512)])
|
||||
def test_silu_and_mul_quantize_to_fp4_grouped(shape: tuple[int, int, int]) -> None:
|
||||
torch.manual_seed(42)
|
||||
l, m, k = shape
|
||||
|
||||
x = torch.randn((l, m, k * 2), dtype=torch.bfloat16, device="cuda")
|
||||
mask = torch.randint(1, max(2, m // 2), (l,), dtype=torch.int32, device="cuda")
|
||||
|
||||
ref_y = _silu_and_mul_reference(x)
|
||||
|
||||
tensor_amax = ref_y.abs().amax(dim=(1, 2)).to(torch.float32)
|
||||
y_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
|
||||
ref_output, ref_output_scales = scaled_fp4_grouped_quant(ref_y, y_sf_global, mask)
|
||||
output, output_scales = silu_and_mul_scaled_fp4_grouped_quant(x, y_sf_global, mask)
|
||||
|
||||
output = output.permute(2, 0, 1)
|
||||
ref_output = ref_output.permute(2, 0, 1)
|
||||
|
||||
padded_m = ((m + 128 - 1) // 128) * 128
|
||||
output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1)
|
||||
ref_output_scales = ref_output_scales.permute(5, 2, 4, 0, 1, 3).view(
|
||||
l, padded_m, -1
|
||||
)
|
||||
|
||||
for i in range(l):
|
||||
torch.testing.assert_close(ref_output[i, : mask[i]], output[i, : mask[i]])
|
||||
scale_ref = recover_swizzled_scales(ref_output_scales[i], m, k)
|
||||
scale_ans = recover_swizzled_scales(output_scales[i], m, k)
|
||||
torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]])
|
||||
Reference in New Issue
Block a user