[VLM] Introduce FA4 into ViT for SM100/SM103 (#41344)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -243,6 +243,7 @@ class FlashAttentionForwardSm100:
|
||||
q_sf_interleaved: bool = False,
|
||||
kv_sf_interleaved: bool = False,
|
||||
batch_invariant: bool = False,
|
||||
tmem_load_red_max: bool = True,
|
||||
):
|
||||
# MXFP8 block-scaled attention (see interface._flash_attn_fwd):
|
||||
# qk_blockscaled: Q/K fp8 e4m3 + per-32 UE8M0 scales; QK^T runs as
|
||||
@@ -365,6 +366,14 @@ class FlashAttentionForwardSm100:
|
||||
# despite the literal `is_sm103` name.
|
||||
is_sm103 = self.arch.is_family_of(Arch.sm_103f)
|
||||
self.is_sm103 = is_sm103
|
||||
# sm_103 tcgen05.ld.red.max: the TMEM load of S also returns the row max of each
|
||||
# 32-column chunk, replacing the FMNMX reduction in softmax. The hardware max is
|
||||
# taken over the raw S, so it is only used on tiles that need no masking and when
|
||||
# nothing rewrites S before the max (score_mod / bias); there it equals the
|
||||
# software reduction exactly.
|
||||
self.use_tmem_load_red_max = (
|
||||
tmem_load_red_max and is_sm103 and score_mod is None and not has_bias
|
||||
)
|
||||
# enable_ex2_emu is derived: True if tuning config has freq > 0, else fallback to default logic
|
||||
_default_enable_ex2_emu = (
|
||||
self.head_dim_padded <= 128
|
||||
@@ -3796,6 +3805,18 @@ class FlashAttentionForwardSm100:
|
||||
)
|
||||
thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tSAcc).get_slice(tidx)
|
||||
tStS_t2r = thr_tmem_load.partition_S(tSAcc) # (((32,32),1),1,4)
|
||||
if const_expr(self.use_tmem_load_red_max):
|
||||
tmem_load_red_atom = cute.make_copy_atom(
|
||||
tcgen05.copy.LdRed32x32bOp(
|
||||
tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX
|
||||
),
|
||||
self.qk_acc_dtype,
|
||||
)
|
||||
tiled_tmem_load_red = tcgen05.make_tmem_copy(tmem_load_red_atom, tSAcc)
|
||||
tStS_t2r_red = tiled_tmem_load_red.get_slice(tidx).partition_S(tSAcc)
|
||||
else:
|
||||
tiled_tmem_load_red = None
|
||||
tStS_t2r_red = None
|
||||
|
||||
tmem_store_scale_atom = cute.make_copy_atom(
|
||||
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)), Float32
|
||||
@@ -3976,6 +3997,8 @@ class FlashAttentionForwardSm100:
|
||||
tStScale_r2t=tStScale_r2t,
|
||||
tStP_r2t=tStP_r2t,
|
||||
sScale=sScale,
|
||||
tiled_tmem_load_red=tiled_tmem_load_red,
|
||||
tStS_t2r_red=tStS_t2r_red,
|
||||
stage=stage,
|
||||
batch_idx=batch_idx,
|
||||
head_idx=head_idx,
|
||||
@@ -4286,6 +4309,8 @@ class FlashAttentionForwardSm100:
|
||||
bias_emu_off: Optional[Boolean] = None,
|
||||
pipeline_bias: Optional[pipeline.PipelineAsync] = None,
|
||||
bias_si_consumer_state: Optional[pipeline.PipelineState] = None,
|
||||
tiled_tmem_load_red: Optional[cute.TiledCopy] = None,
|
||||
tStS_t2r_red: Optional[cute.Tensor] = None,
|
||||
) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:
|
||||
"""Perform a single step of the softmax computation on a block of attention scores.
|
||||
|
||||
@@ -4319,7 +4344,23 @@ class FlashAttentionForwardSm100:
|
||||
tSrS_t2r = cute.make_rmem_tensor(
|
||||
thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype
|
||||
)
|
||||
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
|
||||
# Tiles without masking take the row max from the sm_103 reducing TMEM load.
|
||||
use_red_max = const_expr(self.use_tmem_load_red_max and mask_fn is None)
|
||||
tile_row_max = None
|
||||
if const_expr(use_red_max):
|
||||
num_t2r = cute.size(tStS_t2r_red, mode=[2])
|
||||
tSrMax = cute.make_rmem_tensor(
|
||||
cute.make_layout((1, num_t2r)), self.qk_acc_dtype
|
||||
)
|
||||
for i in cutlass.range_constexpr(num_t2r):
|
||||
cute.copy_atom_call(
|
||||
tiled_tmem_load_red,
|
||||
tStS_t2r_red[None, 0, i],
|
||||
(tSrS_t2r[None, 0, i], tSrMax[None, i]),
|
||||
)
|
||||
tile_row_max = softmax._compute_row_max(tSrMax.load())
|
||||
else:
|
||||
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
|
||||
# tSrS_t2r = copy_utils.load_t2r(thr_tmem_load, tScS_shape, tStS_t2r)
|
||||
|
||||
if const_expr(self.has_bias and apply_bias):
|
||||
@@ -4386,7 +4427,9 @@ class FlashAttentionForwardSm100:
|
||||
|
||||
if const_expr(mask_fn is not None):
|
||||
mask_fn(tSrS_t2r, n_block=n_block)
|
||||
row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first)
|
||||
row_max, acc_scale = softmax.update_row_max(
|
||||
tSrS_t2r.load(), is_first, tile_row_max=tile_row_max
|
||||
)
|
||||
|
||||
if const_expr(not is_first):
|
||||
# tSrScale_r2t = cute.make_rmem_tensor(thr_tmem_store_scale.partition_S(tScScale).shape, Float32)
|
||||
|
||||
@@ -107,6 +107,15 @@ def _get_device_num_sms(device: torch.device) -> int:
|
||||
return torch.cuda.get_device_properties(device).multi_processor_count
|
||||
|
||||
|
||||
def _tmem_load_red_max_enabled() -> bool:
|
||||
"""Whether the SM100 forward kernel takes the softmax row max from the sm_103
|
||||
tcgen05.ld.red TMEM load (on by default; ignored on other architectures).
|
||||
|
||||
SGLANG_FA4_TMEM_LOAD_RED_MAX=0 falls back to the FMNMX reduction.
|
||||
"""
|
||||
return os.environ.get("SGLANG_FA4_TMEM_LOAD_RED_MAX", "1") != "0"
|
||||
|
||||
|
||||
def _validate_head_dims(
|
||||
head_dim: int, head_dim_v: int, compute_capability: int, alignment: int
|
||||
) -> None:
|
||||
@@ -1235,6 +1244,7 @@ def _flash_attn_fwd(
|
||||
return out, lse
|
||||
|
||||
batch_invariant = is_batch_invariant()
|
||||
tmem_load_red_max = _tmem_load_red_max_enabled()
|
||||
compile_key = (
|
||||
dtype,
|
||||
head_dim,
|
||||
@@ -1297,6 +1307,7 @@ def _flash_attn_fwd(
|
||||
sfk.ndim if sfk is not None else None,
|
||||
sfv.ndim if sfv is not None else None,
|
||||
batch_invariant,
|
||||
tmem_load_red_max,
|
||||
fa_logging.get_fa_log_level(),
|
||||
)
|
||||
|
||||
@@ -1521,6 +1532,7 @@ def _flash_attn_fwd(
|
||||
q_sf_interleaved=q_sf_interleaved,
|
||||
kv_sf_interleaved=kv_sf_interleaved,
|
||||
batch_invariant=batch_invariant,
|
||||
tmem_load_red_max=tmem_load_red_max,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -317,15 +317,26 @@ class SoftmaxSm100(Softmax):
|
||||
|
||||
@cute.jit
|
||||
def update_row_max(
|
||||
self, acc_S_row: cute.TensorSSA, is_first: int
|
||||
self,
|
||||
acc_S_row: cute.TensorSSA,
|
||||
is_first: int,
|
||||
tile_row_max: Float32 | None = None,
|
||||
) -> Tuple[Float32, Float32]:
|
||||
# tile_row_max: max of acc_S_row computed elsewhere (e.g. by the sm_103
|
||||
# tcgen05.ld.red TMEM load), used instead of reducing acc_S_row here.
|
||||
if cutlass.const_expr(is_first):
|
||||
row_max_new = self._compute_row_max(acc_S_row)
|
||||
if cutlass.const_expr(tile_row_max is None):
|
||||
row_max_new = self._compute_row_max(acc_S_row)
|
||||
else:
|
||||
row_max_new = tile_row_max
|
||||
row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0
|
||||
acc_scale = 0.0
|
||||
else:
|
||||
row_max_old = self.row_max[0]
|
||||
row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old)
|
||||
if cutlass.const_expr(tile_row_max is None):
|
||||
row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old)
|
||||
else:
|
||||
row_max_new = utils.fmax(tile_row_max, row_max_old)
|
||||
row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0
|
||||
acc_scale_ = (row_max_old - row_max_safe) * self.scale_log2
|
||||
acc_scale = cute.math.exp2(acc_scale_, fastmath=True)
|
||||
|
||||
@@ -614,6 +614,11 @@ class VisionFlash4Attention(nn.Module):
|
||||
if forward_metadata is not None:
|
||||
cu_seqlens_gpu = forward_metadata.cu_seqlens
|
||||
max_seqlen = forward_metadata.max_seqlen
|
||||
elif isinstance(cu_seqlens, list):
|
||||
# ViT CUDA graph runners pass [cu_seqlens, max_seqlen]; models without
|
||||
# a runner keep passing tensors even when the graph env var is set.
|
||||
cu_seqlens_gpu = cu_seqlens[0]
|
||||
max_seqlen = cu_seqlens[1]
|
||||
else:
|
||||
cu_seqlens_gpu = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
|
||||
cu_seqlens_gpu = cu_seqlens_gpu.to(dtype=torch.int32).to(q.device)
|
||||
@@ -1272,7 +1277,7 @@ class VisionAttention(nn.Module):
|
||||
|
||||
Platform defaults:
|
||||
- CUDA (Hopper SM90): "fa3"
|
||||
- CUDA (Blackwell SM100): "fa4"
|
||||
- CUDA (Blackwell SM100/SM103): "fa4"
|
||||
- CUDA (other): "triton_attn"
|
||||
- Ascend NPU: "ascend_attn"
|
||||
- Other platforms: device-specific optimized backend or "sdpa"
|
||||
@@ -1290,10 +1295,10 @@ class VisionAttention(nn.Module):
|
||||
elif passed_backend is not None:
|
||||
backend = passed_backend
|
||||
elif is_cuda():
|
||||
major, minor = get_device_capability()
|
||||
major, _ = get_device_capability()
|
||||
if major == 9:
|
||||
backend = "fa3"
|
||||
elif major == 10 and minor != 3:
|
||||
elif major == 10:
|
||||
backend = "fa4"
|
||||
else:
|
||||
backend = "triton_attn"
|
||||
|
||||
@@ -102,7 +102,7 @@ class InternViTCudaGraphRunner:
|
||||
|
||||
if backend == "triton_attn":
|
||||
cu_ws = [cu, cu_kk, max_len]
|
||||
elif backend == "fa3":
|
||||
elif backend in ("fa3", "fa4"):
|
||||
cu_ws = [cu, max_len]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
@@ -125,7 +125,7 @@ class InternViTCudaGraphRunner:
|
||||
|
||||
if backend == "triton_attn":
|
||||
cu_ws = [cu, cu_kk, max_len]
|
||||
elif backend == "fa3":
|
||||
elif backend in ("fa3", "fa4"):
|
||||
cu_ws = [cu, max_len]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -199,19 +199,25 @@ class ViTCudaGraphRunner:
|
||||
warmup_cu_ws = [cu_window, cu_window_kk, max_window_len]
|
||||
else:
|
||||
warmup_cu_ws = [cu_full, cu_full_kk, max_full_len]
|
||||
if backend == "fa3":
|
||||
warmup_cu_ws = [warmup_cu_ws[0], warmup_cu_ws[2]]
|
||||
warmup_layouts = [warmup_cu_ws]
|
||||
if backend in ("fa3", "fa4"):
|
||||
warmup_layouts = [[warmup_cu_ws[0], warmup_cu_ws[2]]]
|
||||
if backend == "fa4" and self._fullatt_block_indexes:
|
||||
# FA4 JIT-compiles each kernel variant on first use, which must not
|
||||
# happen inside the capture: warm up the windowed and the full layout.
|
||||
warmup_layouts = [[cu_window, max_window_len], [cu_full, max_full_len]]
|
||||
|
||||
warmup_kwargs = dict(
|
||||
cu_seqlens=warmup_cu_ws, output_ws=self.block_ws[graph_key]
|
||||
)
|
||||
warmup_kwargs = dict(output_ws=self.block_ws[graph_key])
|
||||
if position_embeddings is not None:
|
||||
warmup_kwargs["position_embeddings"] = position_embeddings
|
||||
elif rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
|
||||
warmup_kwargs["rotary_pos_emb_cos"] = rotary_pos_emb_cos
|
||||
warmup_kwargs["rotary_pos_emb_sin"] = rotary_pos_emb_sin
|
||||
with torch.no_grad():
|
||||
vit.blocks[0](self.block_input[graph_key], **warmup_kwargs)
|
||||
for layout in warmup_layouts:
|
||||
vit.blocks[0](
|
||||
self.block_input[graph_key], cu_seqlens=layout, **warmup_kwargs
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
with self._capture_context(), torch.cuda.graph(graph):
|
||||
@@ -236,7 +242,7 @@ class ViTCudaGraphRunner:
|
||||
|
||||
if backend == "triton_attn":
|
||||
cu_seq_len_ws = [cu_seqlens_now, cu_seqlens_kk_now, max_len]
|
||||
elif backend == "fa3":
|
||||
elif backend in ("fa3", "fa4"):
|
||||
cu_seq_len_ws = [cu_seqlens_now, max_len]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
|
||||
Reference in New Issue
Block a user