[Gemma4][Attention] Enable Gemma4 FA4 on SM90 with head-dim 512 support (#42019)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Benjamin Truong
2026-10-10 10:40:19 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 55a8dffc87
commit 37678784c6
11 changed files with 485 additions and 25 deletions
@@ -49,6 +49,7 @@ def flash_attn_with_kvcache(
ver=3,
out=None,
max_seqlen_k: Optional[int] = None,
mask_mod=None,
):
"""
If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from
@@ -139,6 +140,8 @@ def flash_attn_with_kvcache(
normalization factor).
"""
if mask_mod is not None and ver != 4:
raise NotImplementedError("mask_mod requires FlashAttention 4")
if ver == 3:
return fa3_flash_attn_with_kvcache(
q,
@@ -208,6 +211,7 @@ def flash_attn_with_kvcache(
pack_gqa=pack_gqa,
sinks=sinks,
score_mod=score_mod,
mask_mod=mask_mod,
aux_tensors=aux_tensors,
sfq=sfq,
sfk=sfk,
@@ -256,7 +260,10 @@ def flash_attn_varlen_func(
rel_bias_prep_cache=None,
ver=3,
out=None,
mask_mod=None,
):
if mask_mod is not None and ver != 4:
raise NotImplementedError("mask_mod requires FlashAttention 4")
if ver == 3:
return fa3_flash_attn_varlen_func(
q,
@@ -311,6 +318,7 @@ def flash_attn_varlen_func(
num_splits=num_splits,
pack_gqa=pack_gqa,
score_mod=score_mod,
mask_mod=mask_mod,
aux_tensors=aux_tensors,
q_descale=q_descale,
k_descale=k_descale,
@@ -2,7 +2,7 @@ from __future__ import annotations
import inspect
import os
from functools import lru_cache
from functools import cache, lru_cache
from typing import Callable, Optional, Tuple, Union
import torch
@@ -232,6 +232,7 @@ def flash_attn_varlen_func(
rel_bias: Optional[torch.Tensor] = None,
rel_bias_prep_cache: Optional[dict] = None,
return_softmax_lse: bool = False,
mask_mod: Optional[Callable] = None,
out: Optional[torch.Tensor] = None,
**_: object,
):
@@ -316,6 +317,7 @@ def flash_attn_varlen_func(
learnable_sink=learnable_sink,
num_splits=num_splits,
pack_gqa=pack_gqa,
mask_mod=mask_mod,
score_mod=score_mod,
aux_tensors=aux_tensors,
return_lse=return_softmax_lse,
@@ -373,6 +375,7 @@ def flash_attn_with_kvcache(
rel_bias: Optional[torch.Tensor] = None,
rel_bias_prep_cache: Optional[dict] = None,
return_softmax_lse: bool = False,
mask_mod: Optional[Callable] = None,
out: Optional[torch.Tensor] = None,
**_: object,
):
@@ -405,6 +408,7 @@ def flash_attn_with_kvcache(
num_splits=num_splits,
pack_gqa=pack_gqa,
learnable_sink=sinks,
mask_mod=mask_mod,
score_mod=score_mod,
aux_tensors=aux_tensors,
q_descale=q_descale,
@@ -424,3 +428,41 @@ def flash_attn_with_kvcache(
if isinstance(result, tuple):
return result[0]
return result
@cache
def make_image_mask_mod(window_left: int = -1):
"""Causal text plus same-image bidirectionality, with a left-window bound.
aux_tensors are packed query image ranges (inclusive, -1 for text) and
cu_seqlens_q. Ranges use absolute positions in each request's KV sequence.
Cache the callback so its identity remains stable across forward passes.
"""
import cutlass
import cutlass.cute as cute
from cutlass import Int32
from sglang.kernels.ops.attention.flash_attn.cute.utils import (
scalar_to_ssa,
ssa_to_scalar,
)
@cute.jit
def image_mask(batch_idx, head_idx, q_idx, kv_idx, seqlen_info, aux_tensors):
ranges, cu_q = aux_tensors
# Partial tiles can call the mask on padded query rows. Clamp the load;
# the kernel separately masks those rows out of the output.
local_q = cutlass.min(ssa_to_scalar(q_idx), seqlen_info.seqlen_q - 1)
row = cutlass.max(cu_q[batch_idx[0]] + local_q, Int32(0))
begin = scalar_to_ssa(ranges[row, 0], Int32)
end = scalar_to_ssa(ranges[row, 1], Int32)
absolute_q = q_idx + scalar_to_ssa(
seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32
)
keep = (kv_idx <= absolute_q) | ((kv_idx >= begin) & (kv_idx <= end))
if cutlass.const_expr(window_left >= 0):
keep = keep & (kv_idx >= absolute_q - window_left)
return keep
image_mask.__vec_size__ = 1
return image_mask
@@ -118,13 +118,14 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
)
def _get_tiled_mma(self):
atom_layout_n = 2 if max(self.tile_hdim, self.tile_hdimv) > 256 else 1
tiled_mma_qk = sm90_utils_basic.make_trivial_tiled_mma(
self.dtype,
self.dtype,
warpgroup.OperandMajorMode.K,
warpgroup.OperandMajorMode.K,
Float32,
atom_layout_mnk=(self.tile_m // 64, 1, 1),
atom_layout_mnk=(self.tile_m // 64, atom_layout_n, 1),
tiler_mn=(64, self.tile_n),
)
tiled_mma_pv = sm90_utils_basic.make_trivial_tiled_mma(
@@ -135,10 +136,10 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
Float32,
atom_layout_mnk=(
self.tile_m // 64,
atom_layout_n,
1,
1,
), # Might need (1, 2, 1) for hdim 512
tiler_mn=(64, self.tile_hdimv),
),
tiler_mn=(64, min(256, self.tile_hdimv)),
a_source=(
warpgroup.OperandSource.RMEM
if self.mma_pv_is_rs
@@ -1857,10 +1858,23 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
if const_expr(not self.mma_pv_is_rs):
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
# Both warp groups consume the same P tile for hdim 512.
# Wait for every reader before overwriting shared memory.
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PEmpty),
number_of_threads=self.num_mma_threads,
)
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
# Fence and barrier to make smem store visible to WGMMA
cute.arch.fence_view_async_shared()
cute.arch.sync_warp()
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PFull),
number_of_threads=self.num_mma_threads,
)
else:
cute.arch.sync_warp()
# For RescaleOBeforeGemm: initialize acc_O
if const_expr(self.rescale_O_before_gemm):
@@ -1962,12 +1976,25 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
utils.cvt_f16(tOrP_acc, tOrP_cur)
if const_expr(not self.mma_pv_is_rs):
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
# Both warp groups consume the same P tile for hdim 512.
# Wait for every reader before overwriting shared memory.
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PEmpty),
number_of_threads=self.num_mma_threads,
)
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
softmax.rescale_O(acc_O, row_scale)
if const_expr(not self.mma_pv_is_rs):
# Fence and barrier to make sure smem store is visible to WGMMA
cute.arch.fence_view_async_shared()
cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PFull),
number_of_threads=self.num_mma_threads,
)
else:
cute.arch.sync_warp()
pipeline_v.consumer_wait(
smem_pipe_read, pipeline_v.consumer_try_wait(smem_pipe_read)
)
@@ -2056,6 +2083,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
utils.cvt_f16(tOrP_acc, tOrP_cur)
if const_expr(not self.mma_pv_is_rs):
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
# Both warp groups consume the same P tile for hdim 512.
# Wait for every reader before overwriting shared memory.
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PEmpty),
number_of_threads=self.num_mma_threads,
)
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
if const_expr(not self.rescale_O_before_gemm):
softmax.rescale_O(acc_O, row_scale)
@@ -2064,7 +2098,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
if const_expr(not self.mma_pv_is_rs):
# Fence and barrier to make sure smem store is visible to WGMMA
cute.arch.fence_view_async_shared()
cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.PFull),
number_of_threads=self.num_mma_threads,
)
else:
cute.arch.sync_warp()
return smem_pipe_read
@cute.jit
@@ -127,13 +127,16 @@ def _validate_head_dims(
is_dedicate_kernel_shape = head_dim == 256 and head_dim_v == 256
is_standard_range = 8 <= head_dim <= 128 and 8 <= head_dim_v <= 128
is_sm90_range = 8 <= head_dim <= 256 and 8 <= head_dim_v <= 256
is_sm90_range = (8 <= head_dim <= 256 and 8 <= head_dim_v <= 256) or (
head_dim == head_dim_v == 512
)
if compute_capability == 9:
assert (
is_sm90_range and head_dim % alignment == 0 and head_dim_v % alignment == 0
), (
f"(head_dim, head_dim_v)=({head_dim}, {head_dim_v}) is not supported on SM90. "
f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}."
f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}, "
"or both equal to 512."
)
elif compute_capability in [10, 11]:
assert (
@@ -209,9 +212,11 @@ def _tile_size_fwd_sm90(
elif head_dim <= 192:
tile_n = 96 if is_local else (128 if head_dim_v <= 128 else 112)
return FwdConfig(128, tile_n, True, True)
else: # hdim 256
elif head_dim <= 256:
tile_n = 64 if is_local else 80
return FwdConfig(128, tile_n, True, True)
else: # hdim 512
return FwdConfig(64, 64, False, True)
def maybe_contiguous(x):
@@ -295,6 +300,22 @@ def num_splits_heuristic(total_mblocks, num_SMs, num_n_blocks, max_splits):
return min(num_SMs // total_mblocks, max_splits, num_n_blocks)
def num_splits_heuristic_sm90_decode(total_mblocks, num_SMs, num_n_blocks, max_splits):
"""Conservative short-span decode policy; all inputs are host-side bounds.
A short span (at most 32 KV tiles) with at least a quarter-SM wave of
independent query/head blocks amortizes poorly across split/reduce launches.
Keep it unsplit. For sparse grids or longer spans retain the upstream
occupancy heuristic. These thresholds are a policy, not an autotuned optimum.
"""
if num_n_blocks <= 32 and total_mblocks * 4 >= num_SMs:
return 1
return max(1, num_splits_heuristic(
total_mblocks, num_SMs, num_n_blocks, max_splits
))
def _resolve_causal_local_window(
causal, window_size_left, window_size_right, mask_mod=None
):
@@ -802,8 +823,8 @@ def _flash_attn_fwd(
0,
min(
max_seqlen_k,
(window_size_right or max_seqlen_k)
+ (window_size_left or max_seqlen_k)
(max_seqlen_k if window_size_right is None else window_size_right)
+ (max_seqlen_k if window_size_left is None else window_size_left)
+ 1
+ tile_m,
),
@@ -860,7 +881,10 @@ def _flash_attn_fwd(
)
num_splits = arch_forward_plan.num_splits
elif num_splits < 1:
num_splits = num_splits_heuristic(
split_selector = num_splits_heuristic
if arch // 10 == 9 and max_seqlen_q == 1:
split_selector = num_splits_heuristic_sm90_decode
num_splits = split_selector(
total_mblocks,
num_SMs,
num_n_blocks,
@@ -1435,8 +1459,7 @@ def _flash_attn_fwd(
pack_gqa=pack_gqa,
tile_m=tile_m,
tile_n=tile_n,
# num_stages=1,
num_stages=2,
num_stages=1 if max(head_dim, head_dim_v) > 256 else 2,
num_threads=num_threads,
Q_in_regs=False,
intra_wg_overlap=intra_wg_overlap,
@@ -93,7 +93,8 @@ class PagedKVManager(ParamsBase):
atom_async_copy, thr_layout, val_layout
)
gmem_thr_copy_KV = gmem_tiled_copy_KV.get_slice(thread_idx)
page_entry_per_thread = n_block_size // num_threads
# Cover short tiles and the remaining rows of a partial final thread group.
page_entry_per_thread = (n_block_size + num_threads - 1) // num_threads
if const_expr(mSFK_paged is not None or mSFV_paged is not None):
atom_async_copy_sf = cute.make_copy_atom(
@@ -299,6 +300,7 @@ class PagedKVManager(ParamsBase):
if n_block >= 0
else 0
)
rows_per_pass = self.num_threads // self.gmem_threads_per_row
for m in cutlass.range_constexpr(cute.size(tXsX, mode=[1])):
row_valid = tXc0X[0, m, 0][0] < seqlenk_row_limit
should_load = cute.make_fragment_like(tXsX[(0, None), m, 0], cute.Boolean)
@@ -319,7 +321,14 @@ class PagedKVManager(ParamsBase):
mX_paged_cur_copy = cute.tiled_divide(
mX_paged_cur, (self.async_copy_elems,)
)
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
if const_expr(tXc0X[0, m, 0][0] + rows_per_pass <= self.n_block_size):
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
else:
# Rows past a partial final pass alias another column block or
# stage. Predicated-off cp.async still zero-fills its destination,
# so out-of-tile threads must not issue a copy.
if tXcX[0, m, 0][0] < self.n_block_size:
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
@cute.jit
def load_sf_KV(self, n_block: Int32, sSFX: cute.Tensor, K_or_V: str):
+2 -1
View File
@@ -643,6 +643,7 @@ def handle_model_specific_adjustments(server_args: Any):
accepted_backends = (
"trtllm_mha",
"triton",
"fa4",
"ascend",
"intel_xpu",
"intel_amx",
@@ -651,7 +652,7 @@ def handle_model_specific_adjustments(server_args: Any):
assert (
prefill_backend in accepted_backends and decode_backend in accepted_backends
), (
"Gemma4 only supports trtllm_mha, triton, ascend, intel_xpu, intel_amx, or "
"Gemma4 only supports trtllm_mha, triton, fa4, ascend, intel_xpu, intel_amx, or "
f"aiter attention backend, got prefill={prefill_backend}, decode={decode_backend}"
)
@@ -93,6 +93,8 @@ class FlashAttentionMetadata:
# CUDA graph (built once per forward, reused across layers). See forward_extend.
fa_skip_cu_seqlens_q: torch.Tensor = None
fa_skip_max_seqlen_q: int = None
# Packed absolute image ranges for Gemma4 prefill; None for text/decode.
image_token_ranges: torch.Tensor = None
# Window size (typically used by Gemma)
window_size: tuple = (-1, -1)
# Page table, the index of KV Cache Tables/Blocks
@@ -1513,6 +1515,33 @@ class FlashAttentionBackend(AttentionBackend):
if score_mod is not None:
kwargs["score_mod"] = score_mod
kwargs["aux_tensors"] = aux_tensors
if metadata.image_token_ranges is not None:
if (
self.fa_impl_ver != 4
or cp_active
or use_local_attn
or use_cascade_attn
or self.use_mla
or self.fa_skip_kv_cache
or metadata.swa_spec_metadata is not None
or metadata is self.full_cg_prefill_metadata
or score_mod is not None
):
raise NotImplementedError(
"FA4 image masks require standard paged MHA prefill without "
"context parallelism, full prefill graphs, speculative metadata, "
"or score modifiers."
)
from sglang.kernels.ops.attention.flash_attention_v4 import (
make_image_mask_mod,
)
kwargs["mask_mod"] = make_image_mask_mod(window_size[0])
kwargs["aux_tensors"] = [metadata.image_token_ranges, metadata.cu_seqlens_q]
# The callback permits future keys within an image. Built-in
# causal/window pruning would discard those keys before the mask.
causal = False
window_size = (-1, -1)
kwargs.update(self._mxfp8_sf_kwargs(layer, forward_batch, q_descale))
if fa_k_descale is not None:
kwargs["k_descale"] = fa_k_descale
@@ -298,6 +298,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
"""
_use_draft_input_embeds = False
_fa4_prefill = False
_backend_can_run_prefill_cuda_graph = None
_captured_attn_metadata_max_bs: Optional[int] = None
dllm_attention = None
@@ -316,6 +317,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
if self.dllm_attention is not None
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
)
prefill_attn_backend = getattr(
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
)
self._fa4_prefill = getattr(prefill_attn_backend, "fa_impl_ver", None) == 4
# --- model flags ----------------------------------------------
self.quant_config = getattr(model_runner.model, "quant_config", None)
self.is_multimodal = model_runner.model_config.is_multimodal
@@ -1288,6 +1293,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
):
return False
# Full replay bypasses forward_extend's Python image-mask guard.
# Keep FA4 multimodal prefill eager until its full-graph path is
# supported; both forward dispatch and the DP vote use this policy.
if self._is_full_backend and self._fa4_prefill and contains_mm_inputs:
return False
if self._is_full_backend and batch_size > self._capture_req_slots:
return False
if (
+44 -4
View File
@@ -335,11 +335,51 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
during prefill. Following the HF implementation, bidirectional attention
is only enabled within each individual image group (same-item
tokens), not across items.
Currently only the TritonAttnBackend supports this.
Triton and FA4 use causal attention for image spans split across chunks.
TODO(kpham-sgl): Guard appropriately for gemma3_mm.py:prepare_attn_masks()
"""
if not isinstance(get_attn_backend(), TritonAttnBackend):
attn_backend = get_attn_backend()
prefill_backend = getattr(attn_backend, "prefill_backend", attn_backend)
if getattr(prefill_backend, "fa_impl_ver", None) == 4:
# O(query tokens) metadata instead of a dense Q x KV mask. Keep
# absolute ranges so cached text prefixes retain the right offset.
lengths = forward_batch.extend_seq_lens_cpu
prefixes = forward_batch.extend_prefix_lens_cpu
ranges = torch.full((sum(lengths), 2), -1, dtype=torch.int32)
offset = 0
split_images = []
for i, (length, prefix) in enumerate(zip(lengths, prefixes)):
mm_inputs = forward_batch.mm_inputs[i]
if mm_inputs is not None:
for item in mm_inputs.mm_items:
if not item.is_image():
continue
for begin, end in item.offsets:
if end < prefix or begin >= prefix + length:
continue
if begin < prefix or end >= prefix + length:
# Match Triton: leave split image rows at the
# causal sentinel instead of granting partial
# bidirectional attention within this chunk.
split_images.append((i, begin, end))
continue
ranges[
offset + begin - prefix : offset + end + 1 - prefix
] = torch.tensor([begin, end], dtype=torch.int32)
offset += length
if split_images:
logger.warning_once(
f"{len(split_images)} images are split across chunk boundaries. "
f"First 5 split images (batch, begin, end): {split_images[:5]}. "
"Those images will receive causal attention. Disable chunked "
"prefill (--chunked-prefill-size=-1) for full bidirectional attention."
)
prefill_backend.forward_metadata.image_token_ranges = ranges.to(
input_ids.device, non_blocking=True
)
return
if not isinstance(prefill_backend, TritonAttnBackend):
logger.warning_once(
"Bidirectional attention for image tokens requires TritonAttnBackend. "
"Falling back to causal attention, which may degrade image quality."
@@ -413,10 +453,10 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
)
if bidirectional_attn_masks_list:
bidirectional_attn_masks = torch.cat(bidirectional_attn_masks_list, dim=0)
get_attn_backend().forward_metadata.mask_indptr = (
prefill_backend.forward_metadata.mask_indptr = (
bidirectional_attn_mask_indptr
)
get_attn_backend().forward_metadata.custom_mask = bidirectional_attn_masks
prefill_backend.forward_metadata.custom_mask = bidirectional_attn_masks
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
vt = self.vision_tower