[VLM] Introduce FA4 into ViT for SM100/SM103 (#41344)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2026-09-27 23:04:21 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 588049fe2e
commit c311bc9611
9 changed files with 589 additions and 17 deletions
@@ -243,6 +243,7 @@ class FlashAttentionForwardSm100:
q_sf_interleaved: bool = False,
kv_sf_interleaved: bool = False,
batch_invariant: bool = False,
tmem_load_red_max: bool = True,
):
# MXFP8 block-scaled attention (see interface._flash_attn_fwd):
# qk_blockscaled: Q/K fp8 e4m3 + per-32 UE8M0 scales; QK^T runs as
@@ -365,6 +366,14 @@ class FlashAttentionForwardSm100:
# despite the literal `is_sm103` name.
is_sm103 = self.arch.is_family_of(Arch.sm_103f)
self.is_sm103 = is_sm103
# sm_103 tcgen05.ld.red.max: the TMEM load of S also returns the row max of each
# 32-column chunk, replacing the FMNMX reduction in softmax. The hardware max is
# taken over the raw S, so it is only used on tiles that need no masking and when
# nothing rewrites S before the max (score_mod / bias); there it equals the
# software reduction exactly.
self.use_tmem_load_red_max = (
tmem_load_red_max and is_sm103 and score_mod is None and not has_bias
)
# enable_ex2_emu is derived: True if tuning config has freq > 0, else fallback to default logic
_default_enable_ex2_emu = (
self.head_dim_padded <= 128
@@ -3796,6 +3805,18 @@ class FlashAttentionForwardSm100:
)
thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tSAcc).get_slice(tidx)
tStS_t2r = thr_tmem_load.partition_S(tSAcc) # (((32,32),1),1,4)
if const_expr(self.use_tmem_load_red_max):
tmem_load_red_atom = cute.make_copy_atom(
tcgen05.copy.LdRed32x32bOp(
tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX
),
self.qk_acc_dtype,
)
tiled_tmem_load_red = tcgen05.make_tmem_copy(tmem_load_red_atom, tSAcc)
tStS_t2r_red = tiled_tmem_load_red.get_slice(tidx).partition_S(tSAcc)
else:
tiled_tmem_load_red = None
tStS_t2r_red = None
tmem_store_scale_atom = cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)), Float32
@@ -3976,6 +3997,8 @@ class FlashAttentionForwardSm100:
tStScale_r2t=tStScale_r2t,
tStP_r2t=tStP_r2t,
sScale=sScale,
tiled_tmem_load_red=tiled_tmem_load_red,
tStS_t2r_red=tStS_t2r_red,
stage=stage,
batch_idx=batch_idx,
head_idx=head_idx,
@@ -4286,6 +4309,8 @@ class FlashAttentionForwardSm100:
bias_emu_off: Optional[Boolean] = None,
pipeline_bias: Optional[pipeline.PipelineAsync] = None,
bias_si_consumer_state: Optional[pipeline.PipelineState] = None,
tiled_tmem_load_red: Optional[cute.TiledCopy] = None,
tStS_t2r_red: Optional[cute.Tensor] = None,
) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:
"""Perform a single step of the softmax computation on a block of attention scores.
@@ -4319,7 +4344,23 @@ class FlashAttentionForwardSm100:
tSrS_t2r = cute.make_rmem_tensor(
thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype
)
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
# Tiles without masking take the row max from the sm_103 reducing TMEM load.
use_red_max = const_expr(self.use_tmem_load_red_max and mask_fn is None)
tile_row_max = None
if const_expr(use_red_max):
num_t2r = cute.size(tStS_t2r_red, mode=[2])
tSrMax = cute.make_rmem_tensor(
cute.make_layout((1, num_t2r)), self.qk_acc_dtype
)
for i in cutlass.range_constexpr(num_t2r):
cute.copy_atom_call(
tiled_tmem_load_red,
tStS_t2r_red[None, 0, i],
(tSrS_t2r[None, 0, i], tSrMax[None, i]),
)
tile_row_max = softmax._compute_row_max(tSrMax.load())
else:
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
# tSrS_t2r = copy_utils.load_t2r(thr_tmem_load, tScS_shape, tStS_t2r)
if const_expr(self.has_bias and apply_bias):
@@ -4386,7 +4427,9 @@ class FlashAttentionForwardSm100:
if const_expr(mask_fn is not None):
mask_fn(tSrS_t2r, n_block=n_block)
row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first)
row_max, acc_scale = softmax.update_row_max(
tSrS_t2r.load(), is_first, tile_row_max=tile_row_max
)
if const_expr(not is_first):
# tSrScale_r2t = cute.make_rmem_tensor(thr_tmem_store_scale.partition_S(tScScale).shape, Float32)
@@ -107,6 +107,15 @@ def _get_device_num_sms(device: torch.device) -> int:
return torch.cuda.get_device_properties(device).multi_processor_count
def _tmem_load_red_max_enabled() -> bool:
"""Whether the SM100 forward kernel takes the softmax row max from the sm_103
tcgen05.ld.red TMEM load (on by default; ignored on other architectures).
SGLANG_FA4_TMEM_LOAD_RED_MAX=0 falls back to the FMNMX reduction.
"""
return os.environ.get("SGLANG_FA4_TMEM_LOAD_RED_MAX", "1") != "0"
def _validate_head_dims(
head_dim: int, head_dim_v: int, compute_capability: int, alignment: int
) -> None:
@@ -1235,6 +1244,7 @@ def _flash_attn_fwd(
return out, lse
batch_invariant = is_batch_invariant()
tmem_load_red_max = _tmem_load_red_max_enabled()
compile_key = (
dtype,
head_dim,
@@ -1297,6 +1307,7 @@ def _flash_attn_fwd(
sfk.ndim if sfk is not None else None,
sfv.ndim if sfv is not None else None,
batch_invariant,
tmem_load_red_max,
fa_logging.get_fa_log_level(),
)
@@ -1521,6 +1532,7 @@ def _flash_attn_fwd(
q_sf_interleaved=q_sf_interleaved,
kv_sf_interleaved=kv_sf_interleaved,
batch_invariant=batch_invariant,
tmem_load_red_max=tmem_load_red_max,
)
),
)
@@ -317,15 +317,26 @@ class SoftmaxSm100(Softmax):
@cute.jit
def update_row_max(
self, acc_S_row: cute.TensorSSA, is_first: int
self,
acc_S_row: cute.TensorSSA,
is_first: int,
tile_row_max: Float32 | None = None,
) -> Tuple[Float32, Float32]:
# tile_row_max: max of acc_S_row computed elsewhere (e.g. by the sm_103
# tcgen05.ld.red TMEM load), used instead of reducing acc_S_row here.
if cutlass.const_expr(is_first):
row_max_new = self._compute_row_max(acc_S_row)
if cutlass.const_expr(tile_row_max is None):
row_max_new = self._compute_row_max(acc_S_row)
else:
row_max_new = tile_row_max
row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0
acc_scale = 0.0
else:
row_max_old = self.row_max[0]
row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old)
if cutlass.const_expr(tile_row_max is None):
row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old)
else:
row_max_new = utils.fmax(tile_row_max, row_max_old)
row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0
acc_scale_ = (row_max_old - row_max_safe) * self.scale_log2
acc_scale = cute.math.exp2(acc_scale_, fastmath=True)
+8 -3
View File
@@ -614,6 +614,11 @@ class VisionFlash4Attention(nn.Module):
if forward_metadata is not None:
cu_seqlens_gpu = forward_metadata.cu_seqlens
max_seqlen = forward_metadata.max_seqlen
elif isinstance(cu_seqlens, list):
# ViT CUDA graph runners pass [cu_seqlens, max_seqlen]; models without
# a runner keep passing tensors even when the graph env var is set.
cu_seqlens_gpu = cu_seqlens[0]
max_seqlen = cu_seqlens[1]
else:
cu_seqlens_gpu = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
cu_seqlens_gpu = cu_seqlens_gpu.to(dtype=torch.int32).to(q.device)
@@ -1272,7 +1277,7 @@ class VisionAttention(nn.Module):
Platform defaults:
- CUDA (Hopper SM90): "fa3"
- CUDA (Blackwell SM100): "fa4"
- CUDA (Blackwell SM100/SM103): "fa4"
- CUDA (other): "triton_attn"
- Ascend NPU: "ascend_attn"
- Other platforms: device-specific optimized backend or "sdpa"
@@ -1290,10 +1295,10 @@ class VisionAttention(nn.Module):
elif passed_backend is not None:
backend = passed_backend
elif is_cuda():
major, minor = get_device_capability()
major, _ = get_device_capability()
if major == 9:
backend = "fa3"
elif major == 10 and minor != 3:
elif major == 10:
backend = "fa4"
else:
backend = "triton_attn"
@@ -102,7 +102,7 @@ class InternViTCudaGraphRunner:
if backend == "triton_attn":
cu_ws = [cu, cu_kk, max_len]
elif backend == "fa3":
elif backend in ("fa3", "fa4"):
cu_ws = [cu, max_len]
else:
raise RuntimeError(
@@ -125,7 +125,7 @@ class InternViTCudaGraphRunner:
if backend == "triton_attn":
cu_ws = [cu, cu_kk, max_len]
elif backend == "fa3":
elif backend in ("fa3", "fa4"):
cu_ws = [cu, max_len]
else:
raise RuntimeError(
@@ -199,19 +199,25 @@ class ViTCudaGraphRunner:
warmup_cu_ws = [cu_window, cu_window_kk, max_window_len]
else:
warmup_cu_ws = [cu_full, cu_full_kk, max_full_len]
if backend == "fa3":
warmup_cu_ws = [warmup_cu_ws[0], warmup_cu_ws[2]]
warmup_layouts = [warmup_cu_ws]
if backend in ("fa3", "fa4"):
warmup_layouts = [[warmup_cu_ws[0], warmup_cu_ws[2]]]
if backend == "fa4" and self._fullatt_block_indexes:
# FA4 JIT-compiles each kernel variant on first use, which must not
# happen inside the capture: warm up the windowed and the full layout.
warmup_layouts = [[cu_window, max_window_len], [cu_full, max_full_len]]
warmup_kwargs = dict(
cu_seqlens=warmup_cu_ws, output_ws=self.block_ws[graph_key]
)
warmup_kwargs = dict(output_ws=self.block_ws[graph_key])
if position_embeddings is not None:
warmup_kwargs["position_embeddings"] = position_embeddings
elif rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
warmup_kwargs["rotary_pos_emb_cos"] = rotary_pos_emb_cos
warmup_kwargs["rotary_pos_emb_sin"] = rotary_pos_emb_sin
with torch.no_grad():
vit.blocks[0](self.block_input[graph_key], **warmup_kwargs)
for layout in warmup_layouts:
vit.blocks[0](
self.block_input[graph_key], cu_seqlens=layout, **warmup_kwargs
)
torch.cuda.synchronize()
with self._capture_context(), torch.cuda.graph(graph):
@@ -236,7 +242,7 @@ class ViTCudaGraphRunner:
if backend == "triton_attn":
cu_seq_len_ws = [cu_seqlens_now, cu_seqlens_kk_now, max_len]
elif backend == "fa3":
elif backend in ("fa3", "fa4"):
cu_seq_len_ws = [cu_seqlens_now, max_len]
else:
raise RuntimeError(