From 37678784c6377267e5ef3cf56570d3cba441dab0 Mon Sep 17 00:00:00 2001 From: Benjamin Truong Date: Sat, 10 Oct 2026 09:40:19 +0700 Subject: [PATCH] [Gemma4][Attention] Enable Gemma4 FA4 on SM90 with head-dim 512 support (#42019) Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../kernels/ops/attention/flash_attention.py | 8 + .../ops/attention/flash_attention_v4.py | 44 ++++- .../flash_attn/cute/flash_fwd_sm90.py | 54 +++++- .../attention/flash_attn/cute/interface.py | 39 +++- .../ops/attention/flash_attn/cute/paged_kv.py | 13 +- python/sglang/srt/arg_groups/model_hook.py | 3 +- .../attention/flashattention_backend.py | 29 +++ .../runner/prefill_cuda_graph_runner.py | 10 + python/sglang/srt/models/gemma4_mm.py | 48 ++++- .../ops/attention/test_flash_attention_4.py | 181 +++++++++++++++++- .../test_prefill_cuda_graph_runner.py | 81 ++++++++ 11 files changed, 485 insertions(+), 25 deletions(-) diff --git a/python/sglang/kernels/ops/attention/flash_attention.py b/python/sglang/kernels/ops/attention/flash_attention.py index 2a146ce884..fc39bc7375 100644 --- a/python/sglang/kernels/ops/attention/flash_attention.py +++ b/python/sglang/kernels/ops/attention/flash_attention.py @@ -49,6 +49,7 @@ def flash_attn_with_kvcache( ver=3, out=None, max_seqlen_k: Optional[int] = None, + mask_mod=None, ): """ If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from @@ -139,6 +140,8 @@ def flash_attn_with_kvcache( normalization factor). """ + if mask_mod is not None and ver != 4: + raise NotImplementedError("mask_mod requires FlashAttention 4") if ver == 3: return fa3_flash_attn_with_kvcache( q, @@ -208,6 +211,7 @@ def flash_attn_with_kvcache( pack_gqa=pack_gqa, sinks=sinks, score_mod=score_mod, + mask_mod=mask_mod, aux_tensors=aux_tensors, sfq=sfq, sfk=sfk, @@ -256,7 +260,10 @@ def flash_attn_varlen_func( rel_bias_prep_cache=None, ver=3, out=None, + mask_mod=None, ): + if mask_mod is not None and ver != 4: + raise NotImplementedError("mask_mod requires FlashAttention 4") if ver == 3: return fa3_flash_attn_varlen_func( q, @@ -311,6 +318,7 @@ def flash_attn_varlen_func( num_splits=num_splits, pack_gqa=pack_gqa, score_mod=score_mod, + mask_mod=mask_mod, aux_tensors=aux_tensors, q_descale=q_descale, k_descale=k_descale, diff --git a/python/sglang/kernels/ops/attention/flash_attention_v4.py b/python/sglang/kernels/ops/attention/flash_attention_v4.py index 20784c818b..d22a68aa76 100644 --- a/python/sglang/kernels/ops/attention/flash_attention_v4.py +++ b/python/sglang/kernels/ops/attention/flash_attention_v4.py @@ -2,7 +2,7 @@ from __future__ import annotations import inspect import os -from functools import lru_cache +from functools import cache, lru_cache from typing import Callable, Optional, Tuple, Union import torch @@ -232,6 +232,7 @@ def flash_attn_varlen_func( rel_bias: Optional[torch.Tensor] = None, rel_bias_prep_cache: Optional[dict] = None, return_softmax_lse: bool = False, + mask_mod: Optional[Callable] = None, out: Optional[torch.Tensor] = None, **_: object, ): @@ -316,6 +317,7 @@ def flash_attn_varlen_func( learnable_sink=learnable_sink, num_splits=num_splits, pack_gqa=pack_gqa, + mask_mod=mask_mod, score_mod=score_mod, aux_tensors=aux_tensors, return_lse=return_softmax_lse, @@ -373,6 +375,7 @@ def flash_attn_with_kvcache( rel_bias: Optional[torch.Tensor] = None, rel_bias_prep_cache: Optional[dict] = None, return_softmax_lse: bool = False, + mask_mod: Optional[Callable] = None, out: Optional[torch.Tensor] = None, **_: object, ): @@ -405,6 +408,7 @@ def flash_attn_with_kvcache( num_splits=num_splits, pack_gqa=pack_gqa, learnable_sink=sinks, + mask_mod=mask_mod, score_mod=score_mod, aux_tensors=aux_tensors, q_descale=q_descale, @@ -424,3 +428,41 @@ def flash_attn_with_kvcache( if isinstance(result, tuple): return result[0] return result + + +@cache +def make_image_mask_mod(window_left: int = -1): + """Causal text plus same-image bidirectionality, with a left-window bound. + + aux_tensors are packed query image ranges (inclusive, -1 for text) and + cu_seqlens_q. Ranges use absolute positions in each request's KV sequence. + Cache the callback so its identity remains stable across forward passes. + """ + import cutlass + import cutlass.cute as cute + from cutlass import Int32 + + from sglang.kernels.ops.attention.flash_attn.cute.utils import ( + scalar_to_ssa, + ssa_to_scalar, + ) + + @cute.jit + def image_mask(batch_idx, head_idx, q_idx, kv_idx, seqlen_info, aux_tensors): + ranges, cu_q = aux_tensors + # Partial tiles can call the mask on padded query rows. Clamp the load; + # the kernel separately masks those rows out of the output. + local_q = cutlass.min(ssa_to_scalar(q_idx), seqlen_info.seqlen_q - 1) + row = cutlass.max(cu_q[batch_idx[0]] + local_q, Int32(0)) + begin = scalar_to_ssa(ranges[row, 0], Int32) + end = scalar_to_ssa(ranges[row, 1], Int32) + absolute_q = q_idx + scalar_to_ssa( + seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32 + ) + keep = (kv_idx <= absolute_q) | ((kv_idx >= begin) & (kv_idx <= end)) + if cutlass.const_expr(window_left >= 0): + keep = keep & (kv_idx >= absolute_q - window_left) + return keep + + image_mask.__vec_size__ = 1 + return image_mask diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm90.py b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm90.py index eaa20680c4..f8ce62d53c 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm90.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_sm90.py @@ -118,13 +118,14 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): ) def _get_tiled_mma(self): + atom_layout_n = 2 if max(self.tile_hdim, self.tile_hdimv) > 256 else 1 tiled_mma_qk = sm90_utils_basic.make_trivial_tiled_mma( self.dtype, self.dtype, warpgroup.OperandMajorMode.K, warpgroup.OperandMajorMode.K, Float32, - atom_layout_mnk=(self.tile_m // 64, 1, 1), + atom_layout_mnk=(self.tile_m // 64, atom_layout_n, 1), tiler_mn=(64, self.tile_n), ) tiled_mma_pv = sm90_utils_basic.make_trivial_tiled_mma( @@ -135,10 +136,10 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): Float32, atom_layout_mnk=( self.tile_m // 64, + atom_layout_n, 1, - 1, - ), # Might need (1, 2, 1) for hdim 512 - tiler_mn=(64, self.tile_hdimv), + ), + tiler_mn=(64, min(256, self.tile_hdimv)), a_source=( warpgroup.OperandSource.RMEM if self.mma_pv_is_rs @@ -1857,10 +1858,23 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): if const_expr(not self.mma_pv_is_rs): tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + # Both warp groups consume the same P tile for hdim 512. + # Wait for every reader before overwriting shared memory. + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PEmpty), + number_of_threads=self.num_mma_threads, + ) cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) # Fence and barrier to make smem store visible to WGMMA cute.arch.fence_view_async_shared() - cute.arch.sync_warp() + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PFull), + number_of_threads=self.num_mma_threads, + ) + else: + cute.arch.sync_warp() # For RescaleOBeforeGemm: initialize acc_O if const_expr(self.rescale_O_before_gemm): @@ -1962,12 +1976,25 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): utils.cvt_f16(tOrP_acc, tOrP_cur) if const_expr(not self.mma_pv_is_rs): tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + # Both warp groups consume the same P tile for hdim 512. + # Wait for every reader before overwriting shared memory. + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PEmpty), + number_of_threads=self.num_mma_threads, + ) cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) softmax.rescale_O(acc_O, row_scale) if const_expr(not self.mma_pv_is_rs): # Fence and barrier to make sure smem store is visible to WGMMA cute.arch.fence_view_async_shared() - cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PFull), + number_of_threads=self.num_mma_threads, + ) + else: + cute.arch.sync_warp() pipeline_v.consumer_wait( smem_pipe_read, pipeline_v.consumer_try_wait(smem_pipe_read) ) @@ -2056,6 +2083,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): utils.cvt_f16(tOrP_acc, tOrP_cur) if const_expr(not self.mma_pv_is_rs): tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur) + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + # Both warp groups consume the same P tile for hdim 512. + # Wait for every reader before overwriting shared memory. + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PEmpty), + number_of_threads=self.num_mma_threads, + ) cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP) if const_expr(not self.rescale_O_before_gemm): softmax.rescale_O(acc_O, row_scale) @@ -2064,7 +2098,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase): if const_expr(not self.mma_pv_is_rs): # Fence and barrier to make sure smem store is visible to WGMMA cute.arch.fence_view_async_shared() - cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV + if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256): + cute.arch.barrier( + barrier_id=int(NamedBarrierFwd.PFull), + number_of_threads=self.num_mma_threads, + ) + else: + cute.arch.sync_warp() return smem_pipe_read @cute.jit diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py index 6e19dbd375..1ba332491a 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py @@ -127,13 +127,16 @@ def _validate_head_dims( is_dedicate_kernel_shape = head_dim == 256 and head_dim_v == 256 is_standard_range = 8 <= head_dim <= 128 and 8 <= head_dim_v <= 128 - is_sm90_range = 8 <= head_dim <= 256 and 8 <= head_dim_v <= 256 + is_sm90_range = (8 <= head_dim <= 256 and 8 <= head_dim_v <= 256) or ( + head_dim == head_dim_v == 512 + ) if compute_capability == 9: assert ( is_sm90_range and head_dim % alignment == 0 and head_dim_v % alignment == 0 ), ( f"(head_dim, head_dim_v)=({head_dim}, {head_dim_v}) is not supported on SM90. " - f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}." + f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}, " + "or both equal to 512." ) elif compute_capability in [10, 11]: assert ( @@ -209,9 +212,11 @@ def _tile_size_fwd_sm90( elif head_dim <= 192: tile_n = 96 if is_local else (128 if head_dim_v <= 128 else 112) return FwdConfig(128, tile_n, True, True) - else: # hdim 256 + elif head_dim <= 256: tile_n = 64 if is_local else 80 return FwdConfig(128, tile_n, True, True) + else: # hdim 512 + return FwdConfig(64, 64, False, True) def maybe_contiguous(x): @@ -295,6 +300,22 @@ def num_splits_heuristic(total_mblocks, num_SMs, num_n_blocks, max_splits): return min(num_SMs // total_mblocks, max_splits, num_n_blocks) + +def num_splits_heuristic_sm90_decode(total_mblocks, num_SMs, num_n_blocks, max_splits): + """Conservative short-span decode policy; all inputs are host-side bounds. + + A short span (at most 32 KV tiles) with at least a quarter-SM wave of + independent query/head blocks amortizes poorly across split/reduce launches. + Keep it unsplit. For sparse grids or longer spans retain the upstream + occupancy heuristic. These thresholds are a policy, not an autotuned optimum. + """ + if num_n_blocks <= 32 and total_mblocks * 4 >= num_SMs: + return 1 + return max(1, num_splits_heuristic( + total_mblocks, num_SMs, num_n_blocks, max_splits + )) + + def _resolve_causal_local_window( causal, window_size_left, window_size_right, mask_mod=None ): @@ -802,8 +823,8 @@ def _flash_attn_fwd( 0, min( max_seqlen_k, - (window_size_right or max_seqlen_k) - + (window_size_left or max_seqlen_k) + (max_seqlen_k if window_size_right is None else window_size_right) + + (max_seqlen_k if window_size_left is None else window_size_left) + 1 + tile_m, ), @@ -860,7 +881,10 @@ def _flash_attn_fwd( ) num_splits = arch_forward_plan.num_splits elif num_splits < 1: - num_splits = num_splits_heuristic( + split_selector = num_splits_heuristic + if arch // 10 == 9 and max_seqlen_q == 1: + split_selector = num_splits_heuristic_sm90_decode + num_splits = split_selector( total_mblocks, num_SMs, num_n_blocks, @@ -1435,8 +1459,7 @@ def _flash_attn_fwd( pack_gqa=pack_gqa, tile_m=tile_m, tile_n=tile_n, - # num_stages=1, - num_stages=2, + num_stages=1 if max(head_dim, head_dim_v) > 256 else 2, num_threads=num_threads, Q_in_regs=False, intra_wg_overlap=intra_wg_overlap, diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/paged_kv.py b/python/sglang/kernels/ops/attention/flash_attn/cute/paged_kv.py index 9b24a4268b..59f2c84f4c 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/paged_kv.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/paged_kv.py @@ -93,7 +93,8 @@ class PagedKVManager(ParamsBase): atom_async_copy, thr_layout, val_layout ) gmem_thr_copy_KV = gmem_tiled_copy_KV.get_slice(thread_idx) - page_entry_per_thread = n_block_size // num_threads + # Cover short tiles and the remaining rows of a partial final thread group. + page_entry_per_thread = (n_block_size + num_threads - 1) // num_threads if const_expr(mSFK_paged is not None or mSFV_paged is not None): atom_async_copy_sf = cute.make_copy_atom( @@ -299,6 +300,7 @@ class PagedKVManager(ParamsBase): if n_block >= 0 else 0 ) + rows_per_pass = self.num_threads // self.gmem_threads_per_row for m in cutlass.range_constexpr(cute.size(tXsX, mode=[1])): row_valid = tXc0X[0, m, 0][0] < seqlenk_row_limit should_load = cute.make_fragment_like(tXsX[(0, None), m, 0], cute.Boolean) @@ -319,7 +321,14 @@ class PagedKVManager(ParamsBase): mX_paged_cur_copy = cute.tiled_divide( mX_paged_cur, (self.async_copy_elems,) ) - self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load) + if const_expr(tXc0X[0, m, 0][0] + rows_per_pass <= self.n_block_size): + self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load) + else: + # Rows past a partial final pass alias another column block or + # stage. Predicated-off cp.async still zero-fills its destination, + # so out-of-tile threads must not issue a copy. + if tXcX[0, m, 0][0] < self.n_block_size: + self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load) @cute.jit def load_sf_KV(self, n_block: Int32, sSFX: cute.Tensor, K_or_V: str): diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 485addec92..1a6ceef1bf 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -643,6 +643,7 @@ def handle_model_specific_adjustments(server_args: Any): accepted_backends = ( "trtllm_mha", "triton", + "fa4", "ascend", "intel_xpu", "intel_amx", @@ -651,7 +652,7 @@ def handle_model_specific_adjustments(server_args: Any): assert ( prefill_backend in accepted_backends and decode_backend in accepted_backends ), ( - "Gemma4 only supports trtllm_mha, triton, ascend, intel_xpu, intel_amx, or " + "Gemma4 only supports trtllm_mha, triton, fa4, ascend, intel_xpu, intel_amx, or " f"aiter attention backend, got prefill={prefill_backend}, decode={decode_backend}" ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 6bdddab6ba..8e18842499 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -93,6 +93,8 @@ class FlashAttentionMetadata: # CUDA graph (built once per forward, reused across layers). See forward_extend. fa_skip_cu_seqlens_q: torch.Tensor = None fa_skip_max_seqlen_q: int = None + # Packed absolute image ranges for Gemma4 prefill; None for text/decode. + image_token_ranges: torch.Tensor = None # Window size (typically used by Gemma) window_size: tuple = (-1, -1) # Page table, the index of KV Cache Tables/Blocks @@ -1513,6 +1515,33 @@ class FlashAttentionBackend(AttentionBackend): if score_mod is not None: kwargs["score_mod"] = score_mod kwargs["aux_tensors"] = aux_tensors + if metadata.image_token_ranges is not None: + if ( + self.fa_impl_ver != 4 + or cp_active + or use_local_attn + or use_cascade_attn + or self.use_mla + or self.fa_skip_kv_cache + or metadata.swa_spec_metadata is not None + or metadata is self.full_cg_prefill_metadata + or score_mod is not None + ): + raise NotImplementedError( + "FA4 image masks require standard paged MHA prefill without " + "context parallelism, full prefill graphs, speculative metadata, " + "or score modifiers." + ) + from sglang.kernels.ops.attention.flash_attention_v4 import ( + make_image_mask_mod, + ) + + kwargs["mask_mod"] = make_image_mask_mod(window_size[0]) + kwargs["aux_tensors"] = [metadata.image_token_ranges, metadata.cu_seqlens_q] + # The callback permits future keys within an image. Built-in + # causal/window pruning would discard those keys before the mask. + causal = False + window_size = (-1, -1) kwargs.update(self._mxfp8_sf_kwargs(layer, forward_batch, q_descale)) if fa_k_descale is not None: kwargs["k_descale"] = fa_k_descale diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index f169bf5bb7..7a11965ca4 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -298,6 +298,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): """ _use_draft_input_embeds = False + _fa4_prefill = False _backend_can_run_prefill_cuda_graph = None _captured_attn_metadata_max_bs: Optional[int] = None dllm_attention = None @@ -316,6 +317,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): if self.dllm_attention is not None else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None) ) + prefill_attn_backend = getattr( + model_runner.attn_backend, "prefill_backend", model_runner.attn_backend + ) + self._fa4_prefill = getattr(prefill_attn_backend, "fa_impl_ver", None) == 4 # --- model flags ---------------------------------------------- self.quant_config = getattr(model_runner.model, "quant_config", None) self.is_multimodal = model_runner.model_config.is_multimodal @@ -1288,6 +1293,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft ): return False + # Full replay bypasses forward_extend's Python image-mask guard. + # Keep FA4 multimodal prefill eager until its full-graph path is + # supported; both forward dispatch and the DP vote use this policy. + if self._is_full_backend and self._fa4_prefill and contains_mm_inputs: + return False if self._is_full_backend and batch_size > self._capture_req_slots: return False if ( diff --git a/python/sglang/srt/models/gemma4_mm.py b/python/sglang/srt/models/gemma4_mm.py index 5d1d73fc37..86a4cdce72 100644 --- a/python/sglang/srt/models/gemma4_mm.py +++ b/python/sglang/srt/models/gemma4_mm.py @@ -335,11 +335,51 @@ class Gemma4ForConditionalGeneration(PreTrainedModel): during prefill. Following the HF implementation, bidirectional attention is only enabled within each individual image group (same-item tokens), not across items. - Currently only the TritonAttnBackend supports this. + Triton and FA4 use causal attention for image spans split across chunks. TODO(kpham-sgl): Guard appropriately for gemma3_mm.py:prepare_attn_masks() """ - if not isinstance(get_attn_backend(), TritonAttnBackend): + attn_backend = get_attn_backend() + prefill_backend = getattr(attn_backend, "prefill_backend", attn_backend) + if getattr(prefill_backend, "fa_impl_ver", None) == 4: + # O(query tokens) metadata instead of a dense Q x KV mask. Keep + # absolute ranges so cached text prefixes retain the right offset. + lengths = forward_batch.extend_seq_lens_cpu + prefixes = forward_batch.extend_prefix_lens_cpu + ranges = torch.full((sum(lengths), 2), -1, dtype=torch.int32) + offset = 0 + split_images = [] + for i, (length, prefix) in enumerate(zip(lengths, prefixes)): + mm_inputs = forward_batch.mm_inputs[i] + if mm_inputs is not None: + for item in mm_inputs.mm_items: + if not item.is_image(): + continue + for begin, end in item.offsets: + if end < prefix or begin >= prefix + length: + continue + if begin < prefix or end >= prefix + length: + # Match Triton: leave split image rows at the + # causal sentinel instead of granting partial + # bidirectional attention within this chunk. + split_images.append((i, begin, end)) + continue + ranges[ + offset + begin - prefix : offset + end + 1 - prefix + ] = torch.tensor([begin, end], dtype=torch.int32) + offset += length + if split_images: + logger.warning_once( + f"{len(split_images)} images are split across chunk boundaries. " + f"First 5 split images (batch, begin, end): {split_images[:5]}. " + "Those images will receive causal attention. Disable chunked " + "prefill (--chunked-prefill-size=-1) for full bidirectional attention." + ) + prefill_backend.forward_metadata.image_token_ranges = ranges.to( + input_ids.device, non_blocking=True + ) + return + if not isinstance(prefill_backend, TritonAttnBackend): logger.warning_once( "Bidirectional attention for image tokens requires TritonAttnBackend. " "Falling back to causal attention, which may degrade image quality." @@ -413,10 +453,10 @@ class Gemma4ForConditionalGeneration(PreTrainedModel): ) if bidirectional_attn_masks_list: bidirectional_attn_masks = torch.cat(bidirectional_attn_masks_list, dim=0) - get_attn_backend().forward_metadata.mask_indptr = ( + prefill_backend.forward_metadata.mask_indptr = ( bidirectional_attn_mask_indptr ) - get_attn_backend().forward_metadata.custom_mask = bidirectional_attn_masks + prefill_backend.forward_metadata.custom_mask = bidirectional_attn_masks def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: vt = self.vision_tower diff --git a/test/registered/kernels/ops/attention/test_flash_attention_4.py b/test/registered/kernels/ops/attention/test_flash_attention_4.py index e63b33db83..d344346b12 100644 --- a/test/registered/kernels/ops/attention/test_flash_attention_4.py +++ b/test/registered/kernels/ops/attention/test_flash_attention_4.py @@ -5,6 +5,7 @@ import itertools import math import sys +import unittest import pytest import torch @@ -15,6 +16,12 @@ from sglang.kernels.ops.attention.flash_attention import ( flash_attn_varlen_func, flash_attn_with_kvcache, ) +from sglang.kernels.ops.attention.flash_attention_v4 import ( + flash_attn_with_kvcache as flash_attn_with_kvcache_v4, +) +from sglang.kernels.ops.attention.flash_attention_v4 import ( + make_image_mask_mod, +) from sglang.kernels.ops.attention.flash_attn.cute import ( clear_shear_bias_workspace, ) @@ -24,11 +31,12 @@ from sglang.kernels.ops.attention.flash_attn.cute.interface import ( ) from sglang.srt.utils import is_sm100_or_sm110_supported from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=300, stage="base-b-kernel-unit", runner_config="1-gpu-large") register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200") -# Skip this test on Hopper machine +# Existing tests below target SM100+; the SM90 regression class has its own guard. skip_condition = torch.cuda.get_device_capability() < (10, 0) @@ -1841,5 +1849,174 @@ def test_flash_attn_hd256_noncontiguous_inputs(strided_input): assert torch.equal(lse, lse_dense) +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 9, + "Requires Hopper", +) +class TestFlashAttention4SM90(CustomTestCase): + def test_paged_attention(self): + cases = ( + (1, 1, 0, True), + (64, 1, 1, True), + (128, 17, 1, True), + (128, 1, 3, True), + (64, 3, 3, False), + ) + for (dim, kv_heads, window), ( + page_size, + q_len, + splits, + pack, + ) in itertools.product(((256, 16, (1023, 0)), (512, 4, (-1, -1))), cases): + with self.subTest( + dim=dim, page=page_size, q_len=q_len, splits=splits, pack=pack + ): + torch.manual_seed(42) + lengths = (max(q_len, 7), 1103) + pages_per_seq = (max(lengths) + page_size - 1) // page_size + table = ( + torch.randperm(2 * pages_per_seq, device="cuda") + .reshape(2, -1) + .int() + ) + q = ( + torch.randn(2 * q_len, 32, dim, device="cuda", dtype=torch.bfloat16) + * 0.3 + ) + k = ( + torch.randn( + 2 * pages_per_seq, + page_size, + kv_heads, + dim, + device="cuda", + dtype=torch.bfloat16, + ) + * 0.3 + ) + v = torch.randn_like(k) + out = flash_attn_with_kvcache_v4( + q, + k, + v, + page_table=table, + cache_seqlens=torch.tensor( + lengths, device="cuda", dtype=torch.int32 + ), + cu_seqlens_q=torch.tensor( + [0, q_len, 2 * q_len], device="cuda", dtype=torch.int32 + ), + max_seqlen_q=q_len, + softmax_scale=1.0, + causal=True, + window_size=window, + num_splits=splits, + pack_gqa=pack, + ) + refs = [] + for batch, length in enumerate(lengths): + kb = k[table[batch].long()].reshape(-1, kv_heads, dim)[:length] + vb = v[table[batch].long()].reshape(-1, kv_heads, dim)[:length] + kb = kb.repeat_interleave(32 // kv_heads, dim=1).float() + vb = vb.repeat_interleave(32 // kv_heads, dim=1).float() + qb = q[batch * q_len : (batch + 1) * q_len].float() + scores = torch.einsum("qhd,khd->hqk", qb, kb) + pos = torch.arange(q_len, device="cuda") + length - q_len + keys = torch.arange(length, device="cuda") + mask = keys[None, :] <= pos[:, None] + if window[0] >= 0: + mask &= keys[None, :] >= pos[:, None] - window[0] + scores.masked_fill_(~mask[None, :, :], -torch.inf) + refs.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), vb)) + torch.testing.assert_close( + out.float(), torch.cat(refs), atol=0.01, rtol=0.01 + ) + + def test_image_mask(self): + """Future tokens are visible only within one image, also after a prefix. + + A second text-only request guards batch leakage. A long first image + crosses the left-window boundary, and uneven query lengths exercise + padded tiles and packed GQA. Dropping mask_mod must fail this test. + """ + for dim, kv_heads, window in ((256, 16, 31), (512, 4, -1)): + for splits, pack in ((1, True), (3, False)): + with self.subTest(dim=dim, splits=splits, pack=pack): + torch.manual_seed(123) + lengths, qlens = (1137, 19), (81, 19) + page_size = 64 + table = torch.randperm(36, device="cuda").reshape(2, 18).int() + q = ( + torch.randn(100, 32, dim, device="cuda", dtype=torch.bfloat16) + * 0.3 + ) + k = ( + torch.randn( + 36, + page_size, + kv_heads, + dim, + device="cuda", + dtype=torch.bfloat16, + ) + * 0.3 + ) + v = torch.randn_like(k) + cu_q = torch.tensor([0, 81, 100], device="cuda", dtype=torch.int32) + ranges = torch.full((100, 2), -1, device="cuda", dtype=torch.int32) + prefix = lengths[0] - qlens[0] + images = ((prefix + 3, prefix + 47), (prefix + 55, prefix + 73)) + for begin, end in images: + ranges[begin - prefix : end + 1 - prefix] = torch.tensor( + [begin, end], device="cuda" + ) + out = flash_attn_with_kvcache( + q, + k, + v, + page_table=table, + cache_seqlens=torch.tensor( + lengths, device="cuda", dtype=torch.int32 + ), + cu_seqlens_q=cu_q, + max_seqlen_q=81, + softmax_scale=1.0, + causal=False, + num_splits=splits, + pack_gqa=pack, + mask_mod=make_image_mask_mod(window), + aux_tensors=[ranges, cu_q], + ver=4, + ) + refs = [] + offset = 0 + for batch, (length, qlen) in enumerate(zip(lengths, qlens)): + kb = k[table[batch].long()].reshape(-1, kv_heads, dim)[:length] + vb = v[table[batch].long()].reshape(-1, kv_heads, dim)[:length] + kb = kb.repeat_interleave(32 // kv_heads, dim=1).float() + vb = vb.repeat_interleave(32 // kv_heads, dim=1).float() + scores = torch.einsum( + "qhd,khd->hqk", q[offset : offset + qlen].float(), kb + ) + pos = torch.arange(qlen, device="cuda") + length - qlen + keys = torch.arange(length, device="cuda") + mask = keys[None, :] <= pos[:, None] + if batch == 0: + for begin, end in images: + mask[(pos >= begin) & (pos <= end), begin : end + 1] = ( + True + ) + if window >= 0: + mask &= keys[None, :] >= pos[:, None] - window + scores.masked_fill_(~mask[None], -torch.inf) + refs.append( + torch.einsum("hqk,khd->qhd", scores.softmax(-1), vb) + ) + offset += qlen + torch.testing.assert_close( + out.float(), torch.cat(refs), atol=0.01, rtol=0.01 + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index 6190a97180..f36d92caa6 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -9,6 +9,14 @@ import torch import sglang.srt.model_executor.model_runner_components.cuda_graph_setup as graph_setup import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.managers.scheduler_components.dp_attn import ( + _local_prefill_cuda_graph_vote, +) from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( @@ -585,6 +593,79 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): [(first, True), (second, True), (second, False)], ) + def test_fa4_multimodal_batch_cannot_replay_text_only_full_graph(self): + """FA4 multimodal prefill must fall back before full graph replay. + + Both the DP vote and forward-time gate reject multimodal batches; + image batches are rejected before image_token_ranges is installed. + """ + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._capture_req_slots = 4 + runner.enable_lora = False + runner._qwen_bcg_hc_sidechannel = False + runner._qwen_bcg_mtp_draft = False + runner.capture_hidden_mode = CaptureHiddenMode.NULL + runner.max_num_tokens = 4 + runner.capture_num_tokens = [4] + runner.max_context_size = None + runner._capture_chunked_prefix = False + runner.has_mha_companion_layers = False + + batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=2, + input_ids=torch.zeros(4, dtype=torch.int64), + req_pool_indices=torch.tensor([0, 1]), + seq_lens=torch.tensor([2, 2]), + out_cache_loc=torch.arange(4), + seq_lens_sum=4, + extend_prefix_lens_cpu=[0, 0], + ) + image = MultimodalInputs( + mm_items=[MultimodalDataItem(modality=Modality.IMAGE, offsets=[(0, 1)])] + ) + audio = MultimodalInputs(mm_items=[MultimodalDataItem(modality=Modality.AUDIO)]) + video = MultimodalInputs(mm_items=[MultimodalDataItem(modality=Modality.VIDEO)]) + for backend, fa4 in ( + (Backend.FULL, True), + (Backend.FULL, False), + (Backend.BREAKABLE, True), + (Backend.TC_PIECEWISE, True), + ): + runner.prefill_backend_name = backend + runner._is_full_backend = backend == Backend.FULL + runner._fa4_prefill = fa4 + # Text -> mixed text/image -> text also checks that the fallback + # is per batch, not a permanent disabling of captured graphs. + for mm_inputs in (None, [None, image], [None, audio], [None, video], None): + with self.subTest(backend=backend, fa4=fa4, mm_inputs=mm_inputs): + batch.mm_inputs = mm_inputs + expected = not ( + backend == Backend.FULL and fa4 and mm_inputs is not None + ) + self.assertEqual(runner.can_run_graph(batch), expected) + schedule_batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=lambda: 2, + extend_num_tokens=4, + input_embeds=None, + replace_embeds=None, + prefix_lens=[0, 0], + return_logprob=False, + multimodal_inputs=mm_inputs, + ) + self.assertEqual( + _local_prefill_cuda_graph_vote( + local_batch=schedule_batch, + prefill_graph_runner=runner, + coordinated_prefill=True, + breakable_prefill=backend == Backend.BREAKABLE, + spec_algorithm=None, + model_config=None, + ), + expected, + ) + def test_prefix_gate_only_applies_to_chunked_prefix_variant(self): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._capture_req_slots = 4