[AMD] Pack Qwen3.5 GDN input projections on ROCm (#39902)

Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com>
Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
zijie
2026-09-22 00:52:40 -07:00
committed by GitHub
co-authored by Zijie Chen jacky.cheng
parent 771c9d782d
commit 6412ad8c64
5 changed files with 573 additions and 5 deletions
@@ -55,6 +55,13 @@ def _unsupported_derived_weight_cache_error(
"online weight updates."
)
if model is not None:
# Model-owned caches can publish the same rank-uniform constraint
# without importing individual model implementations in the updater.
for module in model.modules():
reason = getattr(module, "_derived_weight_cache_error", None)
if reason is not None:
return reason
from sglang.kernels.ops.attention.dsv4.gemm import hpc_bf16xfp32_gemm_enabled
if hpc_bf16xfp32_gemm_enabled():
+151 -5
View File
@@ -133,8 +133,12 @@ _is_cpu = is_cpu()
_is_gfx95 = is_gfx95_supported()
_is_hip = is_hip()
_QWEN3_5_MOE_TEXT_MODEL_TYPES = ("qwen3_5_moe_text", "qwen4_exp_text")
# qwen4_exp shares these classes, but the ROCm packed path is Qwen3.5-only.
_QWEN3_5_ROCM_PACKED_MODEL_TYPES = ("qwen3_5_text", "qwen3_5_moe_text")
_is_xpu = is_xpu()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
if _use_aiter:
from aiter.tuned_gemm import tgemm
_hip_use_alt_stream = get_bool_env_var("SGLANG_ALT_STREAM") and _is_hip
_gdn_use_alt_stream = _is_cuda or (
get_bool_env_var("SGLANG_GDN_QKVZ_BA_ALT_STREAM", "False") and _hip_use_alt_stream
@@ -394,6 +398,10 @@ class Qwen3_5GatedDeltaNet(nn.Module):
self._bind_packed_weight_loaders(self.in_proj_ba)
self._fused_in_proj_weight: Optional[torch.Tensor] = None
self._fused_in_proj_qkvz_width = 0
self._fused_in_proj_ba_width = 0
self._fused_in_proj_scale: Optional[torch.Tensor] = None
self._fused_in_proj_sources = None
self._derived_weight_cache_error = None
self._fused_input_proj_cpu_enabled = LazyValue(
lambda: (
_is_cpu
@@ -653,16 +661,27 @@ class Qwen3_5GatedDeltaNet(nn.Module):
return query, key, value, z, b, a
def finalize_fused_in_proj(self) -> None:
"""Stack in_proj_qkvz + in_proj_ba into one GEMM weight;
the module weights become row views of it,
so weight reload and dtype checks still see them."""
if not _is_cuda or self._fused_in_proj_weight is not None:
"""Prepare one BF16 or FP8 GEMM for both input projections.
BF16 parameters alias the packed rows. FP8 keeps the original layouts
for the separate path and publishes its derived-cache update constraint.
"""
if not (_is_cuda or _use_aiter) or self._fused_in_proj_weight is not None:
return
if (
_use_aiter
and self.config.model_type not in _QWEN3_5_ROCM_PACKED_MODEL_TYPES
):
return
if get_lora().enable_lora or get_lora().lora_paths:
# LoRA wraps the individual Linear modules; the fused GEMM would
# bypass their adapters.
return
if _use_aiter and not get_bool_env_var("SGLANG_QWEN35_PACKED_IN_PROJ", "True"):
return
qkvz, ba = self.in_proj_qkvz, self.in_proj_ba
if _use_aiter and self._finalize_fused_fp8_in_proj():
return
if not (
isinstance(qkvz.quant_method, UnquantizedLinearMethod)
and isinstance(ba.quant_method, UnquantizedLinearMethod)
@@ -678,7 +697,124 @@ class Qwen3_5GatedDeltaNet(nn.Module):
ba.weight.data = fused[self._fused_in_proj_qkvz_width :]
self._fused_in_proj_weight = fused
def _finalize_fused_fp8_in_proj(self) -> bool:
"""Pack loaded Quark per-channel FP8 projections without requantizing."""
from aiter.ops.shuffle import shuffle_weight
from sglang.srt.layers.quantization.fp8_utils import use_aiter_bpreshuffle_gemm
from sglang.srt.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
QuarkW8A8Fp8,
)
projections = (self.in_proj_qkvz, self.in_proj_ba)
if not all(
type(getattr(proj, "scheme", None)) is QuarkW8A8Fp8
and proj.scheme.weight_qscheme == "per_channel"
and proj.scheme.per_token
and proj.input_scale is None
and proj.bias is None
and proj.weight.dtype == torch.float8_e4m3fn
and proj.weight.is_cuda
and proj.weight.t().is_contiguous()
and proj.weight.shape[1] % 16 == 0
and proj.weight_scale.numel() == proj.weight.shape[1]
for proj in projections
):
return False
# Keep the derived-cache update constraint uniform across ranks. With
# PP, an attention-only stage may have no GDN cache to publish it from.
if get_parallel().pp_size != 1:
return False
widths = [proj.weight.shape[1] for proj in projections]
if projections[0].weight.shape[0] != projections[1].weight.shape[0]:
return False
# CK's bpreshuffle GEMM requires N % 64 == 0. TP4 has BA=32,
# so pad only the packed result, preserving the original Linear sizes.
padded_width = (sum(widths) + 63) // 64 * 64
weight = torch.zeros(
(padded_width, projections[0].weight.shape[0]),
dtype=projections[0].weight.dtype,
device=projections[0].weight.device,
)
scale = torch.ones((padded_width, 1), dtype=torch.float32, device=weight.device)
offset = 0
for proj, width in zip(projections, widths):
source = proj.weight.t()
if not use_aiter_bpreshuffle_gemm(width):
source = shuffle_weight(source, (16, 16))
# AITER's shuffle is local to 16 output rows; concatenating aligned
# shuffled blocks is identical to shuffling their concatenation.
weight[offset : offset + width].copy_(source)
scale[offset : offset + width].copy_(proj.weight_scale.view(-1, 1))
offset += width
self._fused_in_proj_weight = weight
self._fused_in_proj_scale = scale
self._fused_in_proj_qkvz_width, self._fused_in_proj_ba_width = widths
self._fused_in_proj_sources = tuple(
(value, value.data_ptr(), None if value.is_inference() else value._version)
for proj in projections
for value in (proj.weight, proj.weight_scale)
)
self._derived_weight_cache_error = (
"Online weight updates are not supported with packed FP8 GDN input "
"projections: captured graphs retain a derived weight/scale buffer. "
"Restart with SGLANG_QWEN35_PACKED_IN_PROJ=0 before updating weights."
)
return True
def _fused_fp8_in_proj_sources_valid(self) -> bool:
current = tuple(
value
for proj in (self.in_proj_qkvz, self.in_proj_ba)
for value in (proj.weight, proj.weight_scale)
)
return all(
value is source
and value.data_ptr() == pointer
and (version is None or value._version == version)
for value, (source, pointer, version) in zip(
current, self._fused_in_proj_sources
)
)
def _forward_input_proj(self, hidden_states: torch.Tensor):
if _use_aiter and self._fused_in_proj_weight is not None:
# Unquantized BF16 projections consume the bf16 side of the fused
# AR+RMSNorm tuple; one aiter GEMM replaces the two separate
# projections. Measured on MI355X the packed GEMM only pays off for
# decode/verify-sized batches, so larger batches keep the separate
# projections below.
x = hidden_states[0] if isinstance(hidden_states, tuple) else hidden_states
if (
x.dtype == torch.bfloat16
and 0 < x.shape[0] <= 64
and not torch.compiler.is_compiling()
):
if self._fused_in_proj_scale is not None:
if self._fused_fp8_in_proj_sources_valid():
from sglang.srt.layers.quantization.fp8_utils import (
apply_fp8_linear,
)
fused_out = apply_fp8_linear(
x,
self._fused_in_proj_weight.t(),
self._fused_in_proj_scale,
use_per_token_if_dynamic=True,
)
split = self._fused_in_proj_qkvz_width
return (
fused_out[:, :split],
fused_out[:, split : split + self._fused_in_proj_ba_width],
)
else:
fused_out = tgemm.mm(
x, self._fused_in_proj_weight, None, otype=x.dtype
)
return (
fused_out[:, : self._fused_in_proj_qkvz_width],
fused_out[:, self._fused_in_proj_qkvz_width :],
)
# AMD/aiter fused AR+RMSNorm+per-group-quant path ships a
# ``(bf16, fp8, scale)`` 3-tuple so the FP8 ``in_proj_qkvz`` can
# consume ``(fp8, scale)`` (skipping its internal quant) while the
@@ -688,7 +824,8 @@ class Qwen3_5GatedDeltaNet(nn.Module):
return self._forward_input_proj_fused_quant_amd(hidden_states)
if (
self._fused_in_proj_weight is not None
not _use_aiter
and self._fused_in_proj_weight is not None
and hidden_states.dtype == torch.bfloat16
# Measured on cuBLAS above ~1k rows:
# the merged (m, 4120) GEMM is ~10% slower than the two separate GEMMs.
@@ -1724,6 +1861,15 @@ class Qwen3_5ForCausalLM(nn.Module):
return self.embed_tokens
def prepare_before_cuda_graph_capture(self, model_runner) -> None:
if _use_aiter and self.config.model_type in _QWEN3_5_ROCM_PACKED_MODEL_TYPES:
packed = 0
for module in self.modules():
if isinstance(module, Qwen3_5GatedDeltaNet):
module.finalize_fused_in_proj()
packed += int(module._fused_in_proj_weight is not None)
logger.info(
"Packed BF16/FP8 GDN input projection enabled for %d layers", packed
)
if self.flashinfer_mnnvl_cutedsl_fusion is None:
return
from sglang.srt.layers.moe.qwen35_flashinfer_fusion import (