From c311bc961164be098bb281b743fdbb59ad35b1ba Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Sun, 27 Sep 2026 08:04:21 -0700 Subject: [PATCH] [VLM] Introduce FA4 into ViT for SM100/SM103 (#41344) Co-authored-by: luoyuan.luo --- .../bench_flash_attention_tmem_red_max.py | 154 ++++++++++ .../flash_attn/cute/flash_fwd_sm100.py | 47 ++- .../attention/flash_attn/cute/interface.py | 12 + .../ops/attention/flash_attn/cute/softmax.py | 17 +- python/sglang/srt/layers/attention/vision.py | 11 +- .../internvl_vit_cuda_graph_runner.py | 4 +- .../srt/multimodal/vit_cuda_graph_runner.py | 20 +- .../attention/test_flash_attention_4_sm103.py | 290 ++++++++++++++++++ .../test_vision_backend_selection.py | 51 +++ 9 files changed, 589 insertions(+), 17 deletions(-) create mode 100644 benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py create mode 100644 test/registered/kernels/ops/attention/test_flash_attention_4_sm103.py diff --git a/benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py b/benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py new file mode 100644 index 0000000000..e64e1c19e1 --- /dev/null +++ b/benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py @@ -0,0 +1,154 @@ +# ViT attention with FA4 on SM103: row max from tcgen05.ld.red +# (SGLANG_FA4_TMEM_LOAD_RED_MAX=1) vs FMNMX (=0). Shapes and call follow +# VisionFlash4Attention (varlen, non-causal, no LSE). Each round CUDA-graph times both +# back to back, in alternating order; the per-round ratio is the headline number. +# Outputs must be bit-identical. +# +# Run on B300 / GB300 (elsewhere the flag is ignored): +# python benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py + +from __future__ import annotations + +import argparse +import os +from typing import Iterable + +import torch + +from sglang.kernels.ops.attention.flash_attn.cute.interface import ( + flash_attn_varlen_func, +) + +RED_MAX_ENV = "SGLANG_FA4_TMEM_LOAD_RED_MAX" + +# images, tokens per image, heads, head dim +SHAPES = [ + (8, 4096, 16, 80), # Qwen2.5-VL + (8, 4096, 16, 72), # Qwen3-VL + (8, 4096, 16, 64), # InternViT, GLM / DeepSeek-V4 ViT + (64, 1024, 16, 80), + (64, 1024, 16, 64), + (1, 32768, 16, 80), # one large image or a video + (1, 32768, 16, 72), + (1, 32768, 16, 64), +] + + +def _bits(t: torch.Tensor) -> torch.Tensor: + int_dtype = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[t.element_size()] + return t.contiguous().view(int_dtype) + + +def time_graph(fn, iters: int, calls_per_graph: int = 5) -> float: + """Mean seconds per call, replaying a CUDA graph of `calls_per_graph` calls.""" + for _ in range(3): + fn() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + with torch.cuda.graph(graph, stream=stream): + for _ in range(calls_per_graph): + fn() + torch.cuda.current_stream().wait_stream(stream) + graph.replay() + torch.cuda.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(iters): + graph.replay() + end.record() + torch.cuda.synchronize() + return start.elapsed_time(end) / 1e3 / (iters * calls_per_graph) + + +def main(argv: Iterable[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--rounds", + type=int, + default=10, + help="Rounds per shape; each round times both variants back to back.", + ) + parser.add_argument( + "--iters", type=int, default=50, help="Graph replays per round." + ) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args(list(argv) if argv is not None else None) + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required") + major, minor = torch.cuda.get_device_capability() + if major != 10: + raise RuntimeError(f"This benchmark is for SM10x. Got {major}.{minor}.") + if minor < 3: + print( + f"Note: SM{major}{minor} has no tcgen05.ld.red; {RED_MAX_ENV} is ignored " + "and both variants run the same kernel." + ) + + torch.manual_seed(args.seed) + device = "cuda" + mismatches = [] + print( + f"{'shape':28s} {'FMNMX us':>9s} {'ld.red us':>9s} {'ld.red TF':>9s} " + f"{'best':>7s} {'paired':>7s} {'wins':>5s} bitwise" + ) + for images, tokens, heads, head_dim in SHAPES: + torch.cuda.empty_cache() + q, k, v = [ + torch.randn( + images * tokens, heads, head_dim, device=device, dtype=torch.bfloat16 + ) + for _ in range(3) + ] + cu_seqlens = torch.arange(images + 1, dtype=torch.int32, device=device) * tokens + kwargs = dict( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=tokens, + max_seqlen_k=tokens, + ) + + outputs, times = {}, {False: [], True: []} + for red_max in (False, True): + os.environ[RED_MAX_ENV] = "1" if red_max else "0" + outputs[red_max] = flash_attn_varlen_func(**kwargs)[0] + # Swap the order every round so clock and thermal drift hit both equally. + for r in range(args.rounds): + for red_max in (False, True) if r % 2 == 0 else (True, False): + os.environ[RED_MAX_ENV] = "1" if red_max else "0" + times[red_max].append( + time_graph(lambda: flash_attn_varlen_func(**kwargs), args.iters) + ) + same = torch.equal(_bits(outputs[True]), _bits(outputs[False])) + name = f"{images} x {tokens} tok, h{heads} d{head_dim}" + if not same: + mismatches.append(name) + t0, t1 = min(times[False]), min(times[True]) + # Ratio of the two variants within each round, then the median over rounds. + paired = sorted(a / c for a, c in zip(times[False], times[True])) + wins = f"{sum(p > 1 for p in paired)}/{len(paired)}" + flops = 4.0 * images * heads * head_dim * tokens * tokens + print( + f"{name:28s} {t0 * 1e6:9.1f} {t1 * 1e6:9.1f} {flops / t1 / 1e12:9.1f} " + f"{t0 / t1:6.3f}x {paired[len(paired) // 2]:6.3f}x {wins:>5s} " + f"{'identical' if same else 'DIFFERENT'}", + flush=True, + ) + del q, k, v, outputs + + os.environ.pop(RED_MAX_ENV, None) + if mismatches: + print(f"\nOutputs differ between the variants for: {', '.join(mismatches)}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm100.py b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm100.py index 07e8192cb2..eae71ceb45 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm100.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm100.py @@ -243,6 +243,7 @@ class FlashAttentionForwardSm100: q_sf_interleaved: bool = False, kv_sf_interleaved: bool = False, batch_invariant: bool = False, + tmem_load_red_max: bool = True, ): # MXFP8 block-scaled attention (see interface._flash_attn_fwd): # qk_blockscaled: Q/K fp8 e4m3 + per-32 UE8M0 scales; QK^T runs as @@ -365,6 +366,14 @@ class FlashAttentionForwardSm100: # despite the literal `is_sm103` name. is_sm103 = self.arch.is_family_of(Arch.sm_103f) self.is_sm103 = is_sm103 + # sm_103 tcgen05.ld.red.max: the TMEM load of S also returns the row max of each + # 32-column chunk, replacing the FMNMX reduction in softmax. The hardware max is + # taken over the raw S, so it is only used on tiles that need no masking and when + # nothing rewrites S before the max (score_mod / bias); there it equals the + # software reduction exactly. + self.use_tmem_load_red_max = ( + tmem_load_red_max and is_sm103 and score_mod is None and not has_bias + ) # enable_ex2_emu is derived: True if tuning config has freq > 0, else fallback to default logic _default_enable_ex2_emu = ( self.head_dim_padded <= 128 @@ -3796,6 +3805,18 @@ class FlashAttentionForwardSm100: ) thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tSAcc).get_slice(tidx) tStS_t2r = thr_tmem_load.partition_S(tSAcc) # (((32,32),1),1,4) + if const_expr(self.use_tmem_load_red_max): + tmem_load_red_atom = cute.make_copy_atom( + tcgen05.copy.LdRed32x32bOp( + tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX + ), + self.qk_acc_dtype, + ) + tiled_tmem_load_red = tcgen05.make_tmem_copy(tmem_load_red_atom, tSAcc) + tStS_t2r_red = tiled_tmem_load_red.get_slice(tidx).partition_S(tSAcc) + else: + tiled_tmem_load_red = None + tStS_t2r_red = None tmem_store_scale_atom = cute.make_copy_atom( tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)), Float32 @@ -3976,6 +3997,8 @@ class FlashAttentionForwardSm100: tStScale_r2t=tStScale_r2t, tStP_r2t=tStP_r2t, sScale=sScale, + tiled_tmem_load_red=tiled_tmem_load_red, + tStS_t2r_red=tStS_t2r_red, stage=stage, batch_idx=batch_idx, head_idx=head_idx, @@ -4286,6 +4309,8 @@ class FlashAttentionForwardSm100: bias_emu_off: Optional[Boolean] = None, pipeline_bias: Optional[pipeline.PipelineAsync] = None, bias_si_consumer_state: Optional[pipeline.PipelineState] = None, + tiled_tmem_load_red: Optional[cute.TiledCopy] = None, + tStS_t2r_red: Optional[cute.Tensor] = None, ) -> Tuple[cute.Int32, cute.Int32, cute.Int32]: """Perform a single step of the softmax computation on a block of attention scores. @@ -4319,7 +4344,23 @@ class FlashAttentionForwardSm100: tSrS_t2r = cute.make_rmem_tensor( thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype ) - cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r) + # Tiles without masking take the row max from the sm_103 reducing TMEM load. + use_red_max = const_expr(self.use_tmem_load_red_max and mask_fn is None) + tile_row_max = None + if const_expr(use_red_max): + num_t2r = cute.size(tStS_t2r_red, mode=[2]) + tSrMax = cute.make_rmem_tensor( + cute.make_layout((1, num_t2r)), self.qk_acc_dtype + ) + for i in cutlass.range_constexpr(num_t2r): + cute.copy_atom_call( + tiled_tmem_load_red, + tStS_t2r_red[None, 0, i], + (tSrS_t2r[None, 0, i], tSrMax[None, i]), + ) + tile_row_max = softmax._compute_row_max(tSrMax.load()) + else: + cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r) # tSrS_t2r = copy_utils.load_t2r(thr_tmem_load, tScS_shape, tStS_t2r) if const_expr(self.has_bias and apply_bias): @@ -4386,7 +4427,9 @@ class FlashAttentionForwardSm100: if const_expr(mask_fn is not None): mask_fn(tSrS_t2r, n_block=n_block) - row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first) + row_max, acc_scale = softmax.update_row_max( + tSrS_t2r.load(), is_first, tile_row_max=tile_row_max + ) if const_expr(not is_first): # tSrScale_r2t = cute.make_rmem_tensor(thr_tmem_store_scale.partition_S(tScScale).shape, Float32) diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py index 746b21c179..b62b4f3eb5 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py @@ -107,6 +107,15 @@ def _get_device_num_sms(device: torch.device) -> int: return torch.cuda.get_device_properties(device).multi_processor_count +def _tmem_load_red_max_enabled() -> bool: + """Whether the SM100 forward kernel takes the softmax row max from the sm_103 + tcgen05.ld.red TMEM load (on by default; ignored on other architectures). + + SGLANG_FA4_TMEM_LOAD_RED_MAX=0 falls back to the FMNMX reduction. + """ + return os.environ.get("SGLANG_FA4_TMEM_LOAD_RED_MAX", "1") != "0" + + def _validate_head_dims( head_dim: int, head_dim_v: int, compute_capability: int, alignment: int ) -> None: @@ -1235,6 +1244,7 @@ def _flash_attn_fwd( return out, lse batch_invariant = is_batch_invariant() + tmem_load_red_max = _tmem_load_red_max_enabled() compile_key = ( dtype, head_dim, @@ -1297,6 +1307,7 @@ def _flash_attn_fwd( sfk.ndim if sfk is not None else None, sfv.ndim if sfv is not None else None, batch_invariant, + tmem_load_red_max, fa_logging.get_fa_log_level(), ) @@ -1521,6 +1532,7 @@ def _flash_attn_fwd( q_sf_interleaved=q_sf_interleaved, kv_sf_interleaved=kv_sf_interleaved, batch_invariant=batch_invariant, + tmem_load_red_max=tmem_load_red_max, ) ), ) diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/softmax.py b/python/sglang/kernels/ops/attention/flash_attn/cute/softmax.py index ce341b60c6..4a2443fd88 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/softmax.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/softmax.py @@ -317,15 +317,26 @@ class SoftmaxSm100(Softmax): @cute.jit def update_row_max( - self, acc_S_row: cute.TensorSSA, is_first: int + self, + acc_S_row: cute.TensorSSA, + is_first: int, + tile_row_max: Float32 | None = None, ) -> Tuple[Float32, Float32]: + # tile_row_max: max of acc_S_row computed elsewhere (e.g. by the sm_103 + # tcgen05.ld.red TMEM load), used instead of reducing acc_S_row here. if cutlass.const_expr(is_first): - row_max_new = self._compute_row_max(acc_S_row) + if cutlass.const_expr(tile_row_max is None): + row_max_new = self._compute_row_max(acc_S_row) + else: + row_max_new = tile_row_max row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0 acc_scale = 0.0 else: row_max_old = self.row_max[0] - row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old) + if cutlass.const_expr(tile_row_max is None): + row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old) + else: + row_max_new = utils.fmax(tile_row_max, row_max_old) row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0 acc_scale_ = (row_max_old - row_max_safe) * self.scale_log2 acc_scale = cute.math.exp2(acc_scale_, fastmath=True) diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index bff80a1f1a..693963080e 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -614,6 +614,11 @@ class VisionFlash4Attention(nn.Module): if forward_metadata is not None: cu_seqlens_gpu = forward_metadata.cu_seqlens max_seqlen = forward_metadata.max_seqlen + elif isinstance(cu_seqlens, list): + # ViT CUDA graph runners pass [cu_seqlens, max_seqlen]; models without + # a runner keep passing tensors even when the graph env var is set. + cu_seqlens_gpu = cu_seqlens[0] + max_seqlen = cu_seqlens[1] else: cu_seqlens_gpu = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device) cu_seqlens_gpu = cu_seqlens_gpu.to(dtype=torch.int32).to(q.device) @@ -1272,7 +1277,7 @@ class VisionAttention(nn.Module): Platform defaults: - CUDA (Hopper SM90): "fa3" - - CUDA (Blackwell SM100): "fa4" + - CUDA (Blackwell SM100/SM103): "fa4" - CUDA (other): "triton_attn" - Ascend NPU: "ascend_attn" - Other platforms: device-specific optimized backend or "sdpa" @@ -1290,10 +1295,10 @@ class VisionAttention(nn.Module): elif passed_backend is not None: backend = passed_backend elif is_cuda(): - major, minor = get_device_capability() + major, _ = get_device_capability() if major == 9: backend = "fa3" - elif major == 10 and minor != 3: + elif major == 10: backend = "fa4" else: backend = "triton_attn" diff --git a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py index 1f60d8cf08..c110912530 100644 --- a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py @@ -102,7 +102,7 @@ class InternViTCudaGraphRunner: if backend == "triton_attn": cu_ws = [cu, cu_kk, max_len] - elif backend == "fa3": + elif backend in ("fa3", "fa4"): cu_ws = [cu, max_len] else: raise RuntimeError( @@ -125,7 +125,7 @@ class InternViTCudaGraphRunner: if backend == "triton_attn": cu_ws = [cu, cu_kk, max_len] - elif backend == "fa3": + elif backend in ("fa3", "fa4"): cu_ws = [cu, max_len] else: raise RuntimeError( diff --git a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py index 4388493f70..808b2b4124 100644 --- a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py @@ -199,19 +199,25 @@ class ViTCudaGraphRunner: warmup_cu_ws = [cu_window, cu_window_kk, max_window_len] else: warmup_cu_ws = [cu_full, cu_full_kk, max_full_len] - if backend == "fa3": - warmup_cu_ws = [warmup_cu_ws[0], warmup_cu_ws[2]] + warmup_layouts = [warmup_cu_ws] + if backend in ("fa3", "fa4"): + warmup_layouts = [[warmup_cu_ws[0], warmup_cu_ws[2]]] + if backend == "fa4" and self._fullatt_block_indexes: + # FA4 JIT-compiles each kernel variant on first use, which must not + # happen inside the capture: warm up the windowed and the full layout. + warmup_layouts = [[cu_window, max_window_len], [cu_full, max_full_len]] - warmup_kwargs = dict( - cu_seqlens=warmup_cu_ws, output_ws=self.block_ws[graph_key] - ) + warmup_kwargs = dict(output_ws=self.block_ws[graph_key]) if position_embeddings is not None: warmup_kwargs["position_embeddings"] = position_embeddings elif rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None: warmup_kwargs["rotary_pos_emb_cos"] = rotary_pos_emb_cos warmup_kwargs["rotary_pos_emb_sin"] = rotary_pos_emb_sin with torch.no_grad(): - vit.blocks[0](self.block_input[graph_key], **warmup_kwargs) + for layout in warmup_layouts: + vit.blocks[0]( + self.block_input[graph_key], cu_seqlens=layout, **warmup_kwargs + ) torch.cuda.synchronize() with self._capture_context(), torch.cuda.graph(graph): @@ -236,7 +242,7 @@ class ViTCudaGraphRunner: if backend == "triton_attn": cu_seq_len_ws = [cu_seqlens_now, cu_seqlens_kk_now, max_len] - elif backend == "fa3": + elif backend in ("fa3", "fa4"): cu_seq_len_ws = [cu_seqlens_now, max_len] else: raise RuntimeError( diff --git a/test/registered/kernels/ops/attention/test_flash_attention_4_sm103.py b/test/registered/kernels/ops/attention/test_flash_attention_4_sm103.py new file mode 100644 index 0000000000..a9099a72e5 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_flash_attention_4_sm103.py @@ -0,0 +1,290 @@ +"""SM103 FA4 tests for the softmax row max from tcgen05.ld.red. + +The output and LSE must match the FMNMX path (SGLANG_FA4_TMEM_LOAD_RED_MAX=0) +bit for bit and stay within FA4's usual tolerance of the reference. +""" + +import math +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func +from sglang.kernels.ops.attention.flash_attn.cute.flash_fwd_sm100 import ( + FlashAttentionForwardSm100, +) +from sglang.kernels.ops.attention.flash_attn.cute.testing import ( + attention_ref, + generate_qkv, + generate_random_padding_mask, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=150, stage="base-c", runner_config="4-gpu-gb300") + +if not ( + torch.cuda.is_available() + and torch.cuda.get_device_capability()[0] == 10 + and torch.cuda.get_device_capability()[1] >= 3 +): + pytest.skip( + "tcgen05.ld.red is only used on SM 10.3+ (B300 / GB300).", + allow_module_level=True, + ) + +RED_MAX_ENV = "SGLANG_FA4_TMEM_LOAD_RED_MAX" + + +def _bits(t: torch.Tensor) -> torch.Tensor: + int_dtype = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[t.element_size()] + return t.contiguous().view(int_dtype) + + +def _run_fa4(monkeypatch, red_max: bool, **kwargs): + # The flag is in the compile key, so both variants coexist. + monkeypatch.setenv(RED_MAX_ENV, "1" if red_max else "0") + return flash_attn_varlen_func(**kwargs, return_softmax_lse=True, ver=4) + + +def _check_red_max( + monkeypatch, + *, + batch_size, + seqlen_q, + seqlen_k, + nheads, + nheads_kv, + d, + dv=None, + dtype=torch.bfloat16, + causal=False, + window_size=(None, None), + softcap=0.0, + has_learnable_sink=False, + pack_gqa=None, + num_splits=1, + page_size=None, +): + dv = dv or d + device = "cuda" + local = window_size != (None, None) + torch.random.manual_seed( + seqlen_q + seqlen_k + d + dv + int(causal) * 2 + int(local) + ) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + q_ref = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + if softcap > 0.0: + # Keep qk within the softcap range. + q_ref = q_ref * softcap / 4 + q_ref = q_ref.to(dtype).to(dtype_ref) + k_ref = ( + torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + v_ref = ( + torch.randn(batch_size, seqlen_k, nheads_kv, dv, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + learnable_sink = ( + torch.randn(nheads, dtype=torch.bfloat16, device=device) + if has_learnable_sink + else None + ) + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [ + torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) * 2 + for _ in range(3) + ] + else: + q_descale, k_descale, v_descale = None, None, None + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random" + ) + key_padding_mask = generate_random_padding_mask( + seqlen_k, batch_size, device, mode="random" + ) + if (causal or local) and seqlen_q == seqlen_k: + key_padding_mask = query_padding_mask + + qkv = generate_qkv(q_ref, k_ref, v_ref, query_padding_mask, key_padding_mask) + q_unpad, k_unpad, v_unpad = [x.detach().to(dtype) for x in qkv[:3]] + cu_seqlens_q, cu_seqlens_k, _, _, max_seqlen_q, max_seqlen_k = qkv[4:10] + output_pad_fn = qkv[14] + + kwargs = dict( + q=q_unpad, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=max_seqlen_q, + causal=causal, + window_size=window_size, + softcap=softcap, + sinks=learnable_sink, + pack_gqa=pack_gqa, + num_splits=num_splits, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + if page_size is None: + kwargs.update( + k=k_unpad, v=v_unpad, cu_seqlens_k=cu_seqlens_k, max_seqlen_k=max_seqlen_k + ) + else: + # Contiguous pages per sequence. + num_pages = math.ceil(seqlen_k / page_size) + k_cache, v_cache = [ + torch.zeros( + batch_size, + num_pages * page_size, + nheads_kv, + x.shape[-1], + device=device, + dtype=dtype, + ) + for x in (k_ref, v_ref) + ] + k_cache[:, :seqlen_k] = k_ref.to(dtype) + v_cache[:, :seqlen_k] = v_ref.to(dtype) + kwargs.update( + k=k_cache.view(batch_size * num_pages, page_size, nheads_kv, d), + v=v_cache.view(batch_size * num_pages, page_size, nheads_kv, dv), + cu_seqlens_k=None, + page_table=torch.arange( + batch_size * num_pages, dtype=torch.int32, device=device + ).view(batch_size, num_pages), + seqused_k=key_padding_mask.sum(-1, dtype=torch.int32), + ) + + out_red, lse_red = _run_fa4(monkeypatch, True, **kwargs) + out_fmnmx, lse_fmnmx = _run_fa4(monkeypatch, False, **kwargs) + assert torch.equal(_bits(out_red), _bits(out_fmnmx)), "output differs from FMNMX" + assert torch.equal(_bits(lse_red), _bits(lse_fmnmx)), "LSE differs from FMNMX" + + ref_kwargs = dict( + causal=causal, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + learnable_sink=learnable_sink, + softcap=softcap, + ) + out_ref, _ = attention_ref( + q_ref, k_ref, v_ref, query_padding_mask, key_padding_mask, **ref_kwargs + ) + out_pt, _ = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + **ref_kwargs, + ) + out = output_pad_fn(out_red.to(dtype_ref)) + err = (out - out_ref).abs() + pt_err = (out_pt - out_ref).abs() + print(f"Output max diff: {err.max().item()}, mean diff: {err.mean().item()}") + print(f"Pytorch max diff: {pt_err.max().item()}, mean diff: {pt_err.mean().item()}") + if dtype == torch.float8_e4m3fn: + # FA4 fp8 max error is far above the emulated reference (also without + # ld.red); its mean error is close, so bound the mean. + assert err.mean().item() <= 3 * pt_err.mean().item() + else: + # Same bound as test_flash_attention_4.py. + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + assert err.max().item() <= rtol * pt_err.max().item() + fwd_atol + + +def test_red_max_only_where_it_is_exact(): + def uses_red_max(**kwargs): + return FlashAttentionForwardSm100(head_dim=128, **kwargs).use_tmem_load_red_max + + assert uses_red_max() + assert not uses_red_max(tmem_load_red_max=False) + # score_mod / bias modify S, so the max of the raw S is not usable. + assert not uses_red_max(score_mod=lambda *args: args[0]) + assert not uses_red_max(has_bias=True) + + +@pytest.mark.parametrize("mha_type", ["mha", "gqa"]) +@pytest.mark.parametrize("causal", [False, True]) +@pytest.mark.parametrize("d", [64, 80, 128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (256, 256), + (307, 1000), + (1023, 1024), + (4096, 4096), + ], +) +def test_red_max_varlen_output(monkeypatch, seqlen_q, seqlen_k, d, causal, mha_type): + if causal: + seqlen_k = seqlen_q + _check_red_max( + monkeypatch, + batch_size=9 if seqlen_q <= 1024 else 2, + seqlen_q=seqlen_q, + seqlen_k=seqlen_k, + nheads=6, + nheads_kv=6 if mha_type == "mha" else 2, + d=d, + causal=causal, + ) + + +@pytest.mark.parametrize( + "case", + [ + dict(dtype=torch.float16, causal=True, nheads_kv=2), + dict(dtype=torch.float8_e4m3fn, causal=True), + dict(has_learnable_sink=True, causal=True), + dict(window_size=(128, 0)), + dict(softcap=15.0, causal=True), + dict(seqlen_q=256, seqlen_k=2048, causal=True), + dict(seqlen_q=64, seqlen_k=4096, num_splits=3), + dict(seqlen_q=4, seqlen_k=2000, nheads=32, nheads_kv=4, pack_gqa=True), + dict(seqlen_q=512, seqlen_k=2048, causal=True, page_size=128), + dict(seqlen_q=512, seqlen_k=2048, causal=True, page_size=64), + dict(d=96, causal=True), + dict(d=192, dv=128, causal=True), + dict(d=192, dv=128, seqlen_q=2048, seqlen_k=2048, batch_size=2), + ], + ids=[ + "fp16", + "fp8_e4m3", + "learnable_sink", + "sliding_window", + "softcap", + "chunked_prefill", + "split_kv", + "pack_gqa_decode", + "paged_kv_page128", + "paged_kv_page64", + "hdim96", + "hdim192_128_causal", + "hdim192_128", + ], +) +def test_red_max_features(monkeypatch, case): + kwargs = dict( + batch_size=5, + seqlen_q=1024, + seqlen_k=1024, + nheads=6, + nheads_kv=6, + d=128, + ) + kwargs.update(case) + _check_red_max(monkeypatch, **kwargs) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/attention/test_vision_backend_selection.py b/test/registered/unit/layers/attention/test_vision_backend_selection.py index 3a91eaed88..be36fe83ca 100644 --- a/test/registered/unit/layers/attention/test_vision_backend_selection.py +++ b/test/registered/unit/layers/attention/test_vision_backend_selection.py @@ -52,6 +52,57 @@ def test_npu_backend_selection_priority( assert backend == expected +@pytest.mark.parametrize( + ("capability", "expected"), + [ + ((9, 0), "fa3"), + ((10, 0), "fa4"), + ((10, 3), "fa4"), + ((12, 0), "triton_attn"), + ], +) +def test_cuda_default_backend_by_capability(monkeypatch, capability, expected): + monkeypatch.setattr(vision, "is_cuda", lambda: True) + monkeypatch.setattr(vision, "get_device_capability", lambda: capability) + monkeypatch.setattr( + vision, "get_platform", lambda: SimpleNamespace(is_blackwell=False) + ) + monkeypatch.setattr( + vision, "get_mm", lambda: SimpleNamespace(mm_attention_backend=None) + ) + + backend = vision.VisionAttention._determine_attention_backend(None, None) + + assert backend == expected + + +@pytest.mark.parametrize("graph_env", ["0", "1"]) +def test_fa4_seqlens_from_vit_cuda_graph_list(monkeypatch, graph_env): + # ViT CUDA graph runners pass [cu_seqlens, max_seqlen]; tensors must keep + # working even when SGLANG_VIT_ENABLE_CUDA_GRAPH is set for another model. + monkeypatch.setenv("SGLANG_VIT_ENABLE_CUDA_GRAPH", graph_env) + monkeypatch.setattr(vision, "_is_cuda", True) + calls = [] + + def fake_flash_attn_func(q, k, v, **kwargs): + calls.append(kwargs) + return q + + monkeypatch.setattr(vision, "flash_attn_func", fake_flash_attn_func, raising=False) + q = torch.randn(6, 2, 8) + cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32) + backend = vision.VisionFlash4Attention() + + backend(q, q, q, cu_seqlens=[cu_seqlens, 4], bsz=1, seq_len=6) + backend(q, q, q, cu_seqlens=cu_seqlens, bsz=1, seq_len=6, max_seqlen=4) + + for kwargs in calls: + assert kwargs["ver"] == 4 + assert torch.equal(kwargs["cu_seqlens_q"], cu_seqlens) + assert kwargs["max_seqlen_q"] == kwargs["max_seqlen_k"] == 4 + assert calls[0]["cu_seqlens_q"] is cu_seqlens + + def test_explicit_backend_without_published_mm_context(monkeypatch, npu_platform): monkeypatch.setattr( vision,