[Perf] [AMD] dsv4: optional e8m0 mxscale GEMM for dense 128x128 fp8 linears (6/8) (#44007)
Co-authored-by: Duyi-Wang <duyi.wang@amd.com> Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
"""Fused RMSNorm + per-token 1x128 fp8 quant with power-of-two (ue8m0) scales.
|
||||
|
||||
The e8m0 counterpart of aiter's ``fused_rms_fp8_group_quant(...,
|
||||
output_unquantized_inp1=True, transpose_scale=True)``: one launch emits the bf16
|
||||
normed row (for the readers that want it unquantized) and the fp8 operand of an
|
||||
e8m0 mxscale GEMM. The scale is ``ceil_pow2(amax / 448)`` (aiter's default
|
||||
RoundUp / DeepSeek-V4 reference ``act_quant(scale_fmt="ue8m0")``), stored as one
|
||||
e8m0 byte per group in column-major order: a contiguous ``[K/128, M]`` buffer
|
||||
viewed as ``[M, K/128]``, the layout ``per_group_quant_hip(transpose_scale=True,
|
||||
scale_type=fp8_e8m0)`` emits and aiter's mxscale bpreshuffle GEMM reads.
|
||||
"""
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
_FP8_MAX = 448.0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_fp8_e8m0_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
y_ptr,
|
||||
q_ptr,
|
||||
s_ptr,
|
||||
M,
|
||||
N,
|
||||
x_stride,
|
||||
y_stride,
|
||||
q_stride,
|
||||
eps,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GROUP: tl.constexpr,
|
||||
NUM_G: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, BLOCK_N)
|
||||
mask = offs < N
|
||||
x = tl.load(x_ptr + row * x_stride + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / N + eps)
|
||||
w = tl.load(w_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
y = (x * rstd * w).to(y_ptr.dtype.element_ty)
|
||||
tl.store(y_ptr + row * y_stride + offs, y, mask=mask)
|
||||
|
||||
# Quantize the rounded bf16 row, as a separate quant of the norm output would.
|
||||
yg = tl.reshape(y.to(tl.float32), (NUM_G, GROUP))
|
||||
amax = tl.max(tl.abs(yg), axis=1)
|
||||
bits = (amax * (1.0 / 448.0)).to(tl.int32, bitcast=True)
|
||||
e = (bits >> 23) & 0xFF
|
||||
e = tl.where(((bits & 0x7FFFFF) != 0) & (e < 0xFF), e + 1, e)
|
||||
inv = tl.where(e > 0, 1.0 / (e << 23).to(tl.float32, bitcast=True), 0.0)
|
||||
q = tl.clamp(yg * inv[:, None], -448.0, 448.0).to(q_ptr.dtype.element_ty)
|
||||
q_offs = tl.reshape(offs, (NUM_G, GROUP))
|
||||
tl.store(
|
||||
q_ptr + row * q_stride + q_offs,
|
||||
q,
|
||||
mask=tl.reshape(mask, (NUM_G, GROUP)),
|
||||
)
|
||||
g = tl.arange(0, NUM_G)
|
||||
tl.store(s_ptr + g * M + row, e.to(tl.uint8), mask=g * GROUP < N)
|
||||
|
||||
|
||||
def rmsnorm_fp8_e8m0_quant(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, group_size: int = 128
|
||||
) -> Tuple[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
|
||||
"""Returns ``((q_fp8 [M, N], scale_e8m0 [M, N/128] column-major bytes), y [M, N])``."""
|
||||
assert x.dim() == 2 and x.stride(-1) == 1
|
||||
m, n = x.shape
|
||||
assert n % group_size == 0
|
||||
num_g = n // group_size
|
||||
y = torch.empty((m, n), dtype=x.dtype, device=x.device)
|
||||
q = torch.empty((m, n), dtype=torch.float8_e4m3fn, device=x.device)
|
||||
s = torch.empty((num_g, m), dtype=torch.float8_e8m0fnu, device=x.device)
|
||||
if m > 0:
|
||||
num_g_pow2 = triton.next_power_of_2(num_g)
|
||||
block_n = num_g_pow2 * group_size
|
||||
num_warps = 4 if block_n <= 2048 else (8 if block_n <= 4096 else 16)
|
||||
_rmsnorm_fp8_e8m0_kernel[(m,)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
q,
|
||||
s.view(torch.uint8),
|
||||
m,
|
||||
n,
|
||||
x.stride(0),
|
||||
y.stride(0),
|
||||
q.stride(0),
|
||||
eps,
|
||||
BLOCK_N=block_n,
|
||||
GROUP=group_size,
|
||||
NUM_G=num_g_pow2,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return (q, s.view(m, num_g)), y
|
||||
@@ -1613,6 +1613,15 @@ class Envs:
|
||||
# only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype
|
||||
# picks the buffer dtype and this switch is inert.
|
||||
SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False)
|
||||
# ROCm gfx950, DeepSeek-V4 128x128 block-fp8 dense linears: run the GEMM as
|
||||
# aiter's e8m0 mxscale bpreshuffle (flydsl) instead of the fp32-scale
|
||||
# CK/asm/triton bpreshuffle. The weight bytes are shared; only an e8m0 copy
|
||||
# of the (power-of-two) weight scale is added at load, and activations are
|
||||
# quantized with power-of-two (ue8m0) scales. Restricted to the (N, K)
|
||||
# shapes in SGLANG_DSV4_DENSE_MXSCALE_SHAPES ("N:K,N:K"; empty = built-in
|
||||
# whitelist of shapes that win in the microbench, "all" = every shape).
|
||||
SGLANG_DSV4_DENSE_MXSCALE = EnvBool(False)
|
||||
SGLANG_DSV4_DENSE_MXSCALE_SHAPES = EnvStr("")
|
||||
# fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the
|
||||
# installed aiter ships it; 0 keeps the asm kernel.
|
||||
SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True)
|
||||
|
||||
@@ -56,11 +56,13 @@ from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_block_fp8_supported,
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
apply_fp8_linear,
|
||||
block_fp8_scale_to_e8m0_128,
|
||||
block_fp8_scale_to_mxfp8_e8m0,
|
||||
can_auto_enable_marlin_fp8,
|
||||
can_serve_block_fp8_as_mxfp8,
|
||||
cutlass_fp8_supported,
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
dense_mxscale_wanted,
|
||||
dispatch_block_fp8_mxfp8_linear,
|
||||
dispatch_w8a8_block_fp8_linear,
|
||||
dispatch_w8a8_mxfp8_linear,
|
||||
@@ -183,6 +185,7 @@ if _use_aiter:
|
||||
ACTIVATION_SCHEMES = ["static", "dynamic"]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_DENSE_MXSCALE_LOGGED: set = set()
|
||||
|
||||
DSV4_DEQUANT_FP4_TABLE = torch.tensor(
|
||||
[
|
||||
@@ -902,6 +905,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
# instead of silently reading a permuted weight.
|
||||
layer.aiter_bpreshuffled = True
|
||||
layer.weight.is_shuffled = True
|
||||
self._maybe_prepare_dense_mxscale(layer)
|
||||
|
||||
if (
|
||||
is_xpu()
|
||||
@@ -926,6 +930,36 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
with torch.no_grad():
|
||||
layer.weight_scale_inv.set_(scale_reordered)
|
||||
|
||||
def _maybe_prepare_dense_mxscale(self, layer: Module) -> None:
|
||||
"""SGLANG_DSV4_DENSE_MXSCALE: give a (16, 16)-preshuffled 128x128 block-fp8
|
||||
weight an e8m0 copy of its scale, which routes aiter_w8a8_block_fp8_linear
|
||||
onto aiter's mxscale bpreshuffle GEMM. Lossless only for power-of-two
|
||||
scales (ue8m0 checkpoints); any other layer stays on the fp32 GEMM. The
|
||||
weight bytes are shared, the copy costs N*K/16384 bytes."""
|
||||
if self.weight_block_size != [128, 128] or self.convert_mxfp8_to_block:
|
||||
return
|
||||
if layer.weight.dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
n, k = layer.weight.shape
|
||||
if not dense_mxscale_wanted(n, k):
|
||||
return
|
||||
scale = layer.weight_scale_inv.data
|
||||
if tuple(scale.shape) != (n // 128, k // 128) or n % 128 or k % 128:
|
||||
return
|
||||
e8m0 = block_fp8_scale_to_e8m0_128(scale)
|
||||
if e8m0 is None:
|
||||
logger.warning(
|
||||
"Dense mxscale: %s keeps the fp32 GEMM (scales are not powers of two)",
|
||||
getattr(layer, "prefix", tuple(layer.weight.shape)),
|
||||
)
|
||||
return
|
||||
layer.weight._mxscale_w_scale = e8m0
|
||||
if (n, k) not in _DENSE_MXSCALE_LOGGED:
|
||||
_DENSE_MXSCALE_LOGGED.add((n, k))
|
||||
logger.info(
|
||||
"Dense mxscale (e8m0 bpreshuffle GEMM) enabled for N=%d K=%d", n, k
|
||||
)
|
||||
|
||||
def _prepare_block_fp8_as_mxfp8(self, layer: Module) -> None:
|
||||
layer.block_fp8_mxfp8_ready = False
|
||||
if getattr(layer, "keep_plain_weight_layout", False):
|
||||
|
||||
@@ -121,6 +121,121 @@ def set_force_ck_w8a8(enabled: bool = True) -> None:
|
||||
_FORCE_CK_W8A8 = enabled
|
||||
|
||||
|
||||
# DeepSeek-V4 dense 128x128 block-fp8 linears on aiter's e8m0 mxscale bpreshuffle
|
||||
# GEMM (SGLANG_DSV4_DENSE_MXSCALE). DeepseekV4 enables it via
|
||||
# set_dense_mxscale(True) before its weights load, like set_force_ck_w8a8.
|
||||
_DENSE_MXSCALE: bool = False
|
||||
|
||||
# (N, K) where the e8m0 GEMM (with aiter's dsv4 mxscale tuned rows) beats the
|
||||
# fp32-scale bpreshuffle GEMM (incl. aiter #6004/#6029 tuned rows) on gfx950,
|
||||
# cold-weight, over the decode M range. See dsv4_perf_opt reports/dense_gemm.md.
|
||||
# (512, 7168) (DSpark draft wkv) is untuned on the e8m0 side and loses at M>=96.
|
||||
DSV4_DENSE_MXSCALE_DEFAULT_SHAPES = frozenset(
|
||||
{
|
||||
(2048, 7168), # wqkv_a
|
||||
(1536, 7168), # wq_a (unfused: DSpark draft)
|
||||
(8192, 1536), # wq_b TP8, indexer wq_b
|
||||
(16384, 1536), # wq_b TP4
|
||||
(65536, 1536), # wq_b attn-TP1 (DP attention)
|
||||
(7168, 2048), # wo_b TP8
|
||||
(7168, 4096), # wo_b TP4
|
||||
(7168, 16384), # wo_b attn-TP1 (DP attention)
|
||||
(6144, 7168), # shared expert w13, TP1
|
||||
(7168, 3072), # shared expert w2, TP1
|
||||
(768, 7168), # shared expert w13, TP8
|
||||
(7168, 384), # shared expert w2, TP8
|
||||
(7168, 21504), # DSpark main_proj
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def aiter_has_dense_mxscale() -> bool:
|
||||
"""Whether the installed aiter dispatches e8m0 block scales in
|
||||
gemm_a8w8_blockscale_bpreshuffle. Older aiter takes the e8m0 bytes as fp32
|
||||
scales and silently returns wrong results."""
|
||||
try:
|
||||
from aiter.ops.gemm_op_a8w8 import get_mxscale_bpreshuffle_config # noqa: F401
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def set_dense_mxscale(enabled: bool = True) -> None:
|
||||
global _DENSE_MXSCALE
|
||||
if enabled and not aiter_has_dense_mxscale():
|
||||
logger.warning(
|
||||
"SGLANG_DSV4_DENSE_MXSCALE is set but the installed aiter has no e8m0 "
|
||||
"mxscale bpreshuffle GEMM; keeping the fp32-scale GEMM."
|
||||
)
|
||||
enabled = False
|
||||
_DENSE_MXSCALE = enabled
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _dense_mxscale_shapes() -> Optional[frozenset]:
|
||||
"""The (N, K) whitelist, or None for every shape."""
|
||||
spec = envs.SGLANG_DSV4_DENSE_MXSCALE_SHAPES.get().strip()
|
||||
if not spec:
|
||||
return DSV4_DENSE_MXSCALE_DEFAULT_SHAPES
|
||||
if spec.lower() == "all":
|
||||
return None
|
||||
shapes = set()
|
||||
for item in spec.split(","):
|
||||
n, k = item.strip().split(":")
|
||||
shapes.add((int(n), int(k)))
|
||||
return frozenset(shapes)
|
||||
|
||||
|
||||
def dense_mxscale_wanted(n: int, k: int) -> bool:
|
||||
if not (_DENSE_MXSCALE and _use_aiter_bpreshuffle_gfx95):
|
||||
return False
|
||||
shapes = _dense_mxscale_shapes()
|
||||
return shapes is None or (n, k) in shapes
|
||||
|
||||
|
||||
def block_fp8_scale_to_e8m0_128(scale: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
"""fp32 128x128 block scales [N/128, K/128] -> e8m0 bytes of the same shape,
|
||||
or None if any scale is not a positive normal power of two (not lossless)."""
|
||||
s = scale.detach().float().contiguous()
|
||||
bits = s.view(torch.int32)
|
||||
exp = (bits >> 23) & 0xFF
|
||||
ok = (s > 0) & ((bits & 0x7FFFFF) == 0) & (exp > 0) & (exp < 0xFF)
|
||||
if not bool(ok.all()):
|
||||
return None
|
||||
return exp.to(torch.uint8).view(torch.float8_e8m0fnu).contiguous()
|
||||
|
||||
|
||||
def is_e8m0_scale(scale: Optional[torch.Tensor]) -> bool:
|
||||
return scale is not None and scale.dtype == torch.float8_e8m0fnu
|
||||
|
||||
|
||||
def dense_mxscale_ready(*linears) -> bool:
|
||||
"""Whether every given linear consumes e8m0 (mxscale) activations, i.e. a
|
||||
shared producer of their fp8 operand should emit ue8m0 scales."""
|
||||
if not _DENSE_MXSCALE:
|
||||
return False
|
||||
for linear in linears:
|
||||
if linear is None:
|
||||
continue
|
||||
weight = getattr(linear, "weight", None)
|
||||
if getattr(weight, "_mxscale_w_scale", None) is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def quant_fp8_e8m0_128(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token 1x128 fp8 quant with ue8m0 scales, the scale bytes column-major
|
||||
([K/128, M] storage viewed as [M, K/128]) as the mxscale GEMM reads them."""
|
||||
return per_group_quant_hip(
|
||||
x,
|
||||
quant_dtype=aiter.dtypes.fp8,
|
||||
group_size=128,
|
||||
transpose_scale=True,
|
||||
scale_type=torch.float8_e8m0fnu,
|
||||
)
|
||||
|
||||
|
||||
def materialize_bpreshuffle_fp8_scale(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Materialize the physical scale layout consumed by gfx95 bpreshuffle GEMM."""
|
||||
return scale.t().contiguous().t() if scale.dim() == 2 else scale
|
||||
@@ -235,6 +350,8 @@ if _use_aiter:
|
||||
)
|
||||
|
||||
aiter_per1x128_quant = get_hip_quant(aiter.QuantType.per_1x128)
|
||||
from aiter.ops.quant import per_group_quant_hip
|
||||
|
||||
from sglang.kernels.ops.gemm.smallm_fp8_gfx950 import (
|
||||
smallm_fp8_gemm,
|
||||
smallm_fp8_gemm_supported,
|
||||
@@ -1474,6 +1591,30 @@ def aiter_w8a8_block_fp8_linear(
|
||||
)
|
||||
return out.to(input.dtype).view(*output_shape)
|
||||
|
||||
# DeepSeek-V4 e8m0 mxscale route (SGLANG_DSV4_DENSE_MXSCALE): same
|
||||
# (16, 16)-preshuffled weight, e8m0 copy of its scale. A producer that
|
||||
# already emitted an fp32-scale operand keeps the fp32 GEMM below.
|
||||
mx_w_scale = getattr(weight, "_mxscale_w_scale", None)
|
||||
if is_e8m0_scale(input_scale) or (mx_w_scale is not None and input_scale is None):
|
||||
if mx_w_scale is None:
|
||||
raise RuntimeError(
|
||||
"e8m0-scaled fp8 activation reached a linear without an e8m0 "
|
||||
f"weight scale (weight {tuple(weight.shape)})"
|
||||
)
|
||||
out_dtype = torch.bfloat16 if input_scale is not None else input_2d.dtype
|
||||
if input_2d.shape[0] == 0:
|
||||
return input_2d.new_empty(output_shape, dtype=out_dtype)
|
||||
if input_scale is None:
|
||||
q_input, x_scale = quant_fp8_e8m0_128(input_2d)
|
||||
else:
|
||||
q_input, x_scale = input_2d, input_scale
|
||||
output = gemm_a8w8_blockscale_bpreshuffle(
|
||||
q_input, weight, x_scale, mx_w_scale, dtype=torch.bfloat16
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.to(dtype=out_dtype).view(*output_shape)
|
||||
|
||||
n, k = weight.shape
|
||||
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
|
||||
@@ -180,6 +180,26 @@ def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]:
|
||||
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
|
||||
|
||||
|
||||
def _x_quant_e8m0(layer) -> bool:
|
||||
"""Whether input_layernorm's fp8 operand goes out with ue8m0 scales: only when
|
||||
every dense projection reading it (wqkv_a, or wq_a + wkv) runs the e8m0 mxscale
|
||||
GEMM (SGLANG_DSV4_DENSE_MXSCALE). Resolved on first use, after weight load."""
|
||||
cached = layer.__dict__.get("_x_quant_e8m0")
|
||||
if cached is None:
|
||||
from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready
|
||||
|
||||
attn = layer.self_attn
|
||||
if getattr(attn, "fuse_wqa_wkv", False):
|
||||
consumers = (getattr(attn, "wqkv_a", None),)
|
||||
else:
|
||||
consumers = (getattr(attn, "wq_a", None), getattr(attn, "wkv", None))
|
||||
cached = all(c is not None for c in consumers) and dense_mxscale_ready(
|
||||
*consumers
|
||||
)
|
||||
layer.__dict__["_x_quant_e8m0"] = cached
|
||||
return cached
|
||||
|
||||
|
||||
def input_norm(
|
||||
layer,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -194,7 +214,10 @@ def input_norm(
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
x_quant, hidden_states = fused_rmsnorm_fp8_quant(
|
||||
hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps
|
||||
hidden_states,
|
||||
layer.input_layernorm.weight,
|
||||
layer.rms_norm_eps,
|
||||
e8m0=_x_quant_e8m0(layer),
|
||||
)
|
||||
return hidden_states, x_quant
|
||||
if layer.fused_rmsnorm_fake_quant:
|
||||
|
||||
@@ -551,7 +551,14 @@ def _apply_wo_a_bf16_matmul(
|
||||
return torch.einsum("tgd,grd->tgr", o, wo_a)
|
||||
|
||||
|
||||
def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps):
|
||||
def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps, e8m0: bool = False):
|
||||
if e8m0:
|
||||
# Consumers run the e8m0 mxscale GEMM (SGLANG_DSV4_DENSE_MXSCALE).
|
||||
from sglang.kernels.ops.layernorm.rmsnorm_fp8_e8m0 import (
|
||||
rmsnorm_fp8_e8m0_quant,
|
||||
)
|
||||
|
||||
return rmsnorm_fp8_e8m0_quant(hidden_states, weight, eps)
|
||||
x_quant, x_bf16, _, _ = fused_rms_fp8_group_quant(
|
||||
hidden_states,
|
||||
weight,
|
||||
@@ -1126,6 +1133,22 @@ class MqaAttentionBase(nn.Module):
|
||||
self._attn_sink_local = sink
|
||||
return self._attn_sink_local[:kernel_num_heads]
|
||||
|
||||
def _q_lora_quant_e8m0(self, with_indexer: bool) -> bool:
|
||||
"""Whether the fused q_norm quant feeds wq_b (and the indexer's wq_b, which
|
||||
reuses it) with ue8m0 scales: only when each consumer runs the e8m0 mxscale
|
||||
GEMM. Resolved on first use, after the weights are processed."""
|
||||
key = "_q_lora_e8m0_idx" if with_indexer else "_q_lora_e8m0"
|
||||
cached = self.__dict__.get(key)
|
||||
if cached is None:
|
||||
from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready
|
||||
|
||||
indexer = getattr(self, "indexer", None) if with_indexer else None
|
||||
cached = dense_mxscale_ready(
|
||||
self.wq_b, getattr(indexer, "wq_b", None) if indexer else None
|
||||
)
|
||||
self.__dict__[key] = cached
|
||||
return cached
|
||||
|
||||
@contextmanager
|
||||
def maybe_use_decode_attn_tp(self, forward_batch: ForwardBatch):
|
||||
ctx = get_cp_decode_attn_tp_ctx()
|
||||
@@ -1897,6 +1920,7 @@ class MQALayer(MqaAttentionBase):
|
||||
q_lora,
|
||||
self.q_norm.weight,
|
||||
self.q_norm.variance_epsilon,
|
||||
e8m0=self._q_lora_quant_e8m0(with_indexer=False),
|
||||
)
|
||||
q, _ = self.wq_b(q_for_wqb)
|
||||
else:
|
||||
@@ -2036,6 +2060,7 @@ class MQALayer(MqaAttentionBase):
|
||||
q_lora,
|
||||
self.q_norm.weight,
|
||||
self.q_norm.variance_epsilon,
|
||||
e8m0=self._q_lora_quant_e8m0(with_indexer=True),
|
||||
)
|
||||
q, _ = self.wq_b(q_for_wqb)
|
||||
else:
|
||||
@@ -4251,10 +4276,15 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
# Module-level toggles default OFF; flipped True here for DSV4
|
||||
if _is_hip:
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope
|
||||
from sglang.srt.layers.quantization.fp8_utils import set_force_ck_w8a8
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
set_dense_mxscale,
|
||||
set_force_ck_w8a8,
|
||||
)
|
||||
|
||||
set_force_ck_w8a8(True)
|
||||
set_batched_rope(True)
|
||||
if envs.SGLANG_DSV4_DENSE_MXSCALE.get():
|
||||
set_dense_mxscale(True)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
|
||||
Reference in New Issue
Block a user