66 lines
1.9 KiB
Python
66 lines
1.9 KiB
Python
"""Shared DeepSeek-V3 benchmark shapes and FP8 input preparation."""
|
|
|
|
from typing import Tuple
|
|
|
|
import torch
|
|
from triton import cdiv
|
|
|
|
|
|
def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
assert x.dim() == 2 and x.size(1) % 128 == 0
|
|
m, n = x.shape
|
|
x_view = x.view(m, -1, 128)
|
|
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
|
|
return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view(
|
|
m, n
|
|
), (x_amax / 448.0).view(m, -1)
|
|
|
|
|
|
def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
assert x.dim() == 2
|
|
m, n = x.shape
|
|
x_padded = torch.zeros(
|
|
(cdiv(m, 128) * 128, cdiv(n, 128) * 128), dtype=x.dtype, device=x.device
|
|
)
|
|
x_padded[:m, :n] = x
|
|
x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128)
|
|
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
|
|
x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn)
|
|
return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view(
|
|
x_view.size(0), x_view.size(2)
|
|
)
|
|
|
|
|
|
def get_weight_shapes(tp_size):
|
|
"""Return the DeepSeek-V3 (N, K) shapes, including TP-sharded projections."""
|
|
# cannot TP
|
|
total = [
|
|
(512 + 64, 7168),
|
|
((128 + 64) * 128, 7168),
|
|
(128 * (128 + 128), 512),
|
|
(7168, 16384),
|
|
(7168, 18432),
|
|
]
|
|
# N can TP
|
|
n_tp = [
|
|
(18432 * 2, 7168),
|
|
((128 + 64) * 128, 7168),
|
|
(128 * (128 + 128), 512),
|
|
(24576, 1536),
|
|
(4096, 7168),
|
|
]
|
|
# K can TP
|
|
k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)]
|
|
|
|
weight_shapes = []
|
|
for t in total:
|
|
weight_shapes.append(t)
|
|
for n_t in n_tp:
|
|
new_t = (n_t[0] // tp_size, n_t[1])
|
|
weight_shapes.append(new_t)
|
|
for k_t in k_tp:
|
|
new_t = (k_t[0], k_t[1] // tp_size)
|
|
weight_shapes.append(new_t)
|
|
|
|
return weight_shapes
|