[Gemma4][Attention] Enable Gemma4 FA4 on SM90 with head-dim 512 support (#42019)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
55a8dffc87
commit
37678784c6
@@ -49,6 +49,7 @@ def flash_attn_with_kvcache(
|
||||
ver=3,
|
||||
out=None,
|
||||
max_seqlen_k: Optional[int] = None,
|
||||
mask_mod=None,
|
||||
):
|
||||
"""
|
||||
If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from
|
||||
@@ -139,6 +140,8 @@ def flash_attn_with_kvcache(
|
||||
normalization factor).
|
||||
"""
|
||||
|
||||
if mask_mod is not None and ver != 4:
|
||||
raise NotImplementedError("mask_mod requires FlashAttention 4")
|
||||
if ver == 3:
|
||||
return fa3_flash_attn_with_kvcache(
|
||||
q,
|
||||
@@ -208,6 +211,7 @@ def flash_attn_with_kvcache(
|
||||
pack_gqa=pack_gqa,
|
||||
sinks=sinks,
|
||||
score_mod=score_mod,
|
||||
mask_mod=mask_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
sfq=sfq,
|
||||
sfk=sfk,
|
||||
@@ -256,7 +260,10 @@ def flash_attn_varlen_func(
|
||||
rel_bias_prep_cache=None,
|
||||
ver=3,
|
||||
out=None,
|
||||
mask_mod=None,
|
||||
):
|
||||
if mask_mod is not None and ver != 4:
|
||||
raise NotImplementedError("mask_mod requires FlashAttention 4")
|
||||
if ver == 3:
|
||||
return fa3_flash_attn_varlen_func(
|
||||
q,
|
||||
@@ -311,6 +318,7 @@ def flash_attn_varlen_func(
|
||||
num_splits=num_splits,
|
||||
pack_gqa=pack_gqa,
|
||||
score_mod=score_mod,
|
||||
mask_mod=mask_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from functools import cache, lru_cache
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -232,6 +232,7 @@ def flash_attn_varlen_func(
|
||||
rel_bias: Optional[torch.Tensor] = None,
|
||||
rel_bias_prep_cache: Optional[dict] = None,
|
||||
return_softmax_lse: bool = False,
|
||||
mask_mod: Optional[Callable] = None,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
**_: object,
|
||||
):
|
||||
@@ -316,6 +317,7 @@ def flash_attn_varlen_func(
|
||||
learnable_sink=learnable_sink,
|
||||
num_splits=num_splits,
|
||||
pack_gqa=pack_gqa,
|
||||
mask_mod=mask_mod,
|
||||
score_mod=score_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
return_lse=return_softmax_lse,
|
||||
@@ -373,6 +375,7 @@ def flash_attn_with_kvcache(
|
||||
rel_bias: Optional[torch.Tensor] = None,
|
||||
rel_bias_prep_cache: Optional[dict] = None,
|
||||
return_softmax_lse: bool = False,
|
||||
mask_mod: Optional[Callable] = None,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
**_: object,
|
||||
):
|
||||
@@ -405,6 +408,7 @@ def flash_attn_with_kvcache(
|
||||
num_splits=num_splits,
|
||||
pack_gqa=pack_gqa,
|
||||
learnable_sink=sinks,
|
||||
mask_mod=mask_mod,
|
||||
score_mod=score_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
q_descale=q_descale,
|
||||
@@ -424,3 +428,41 @@ def flash_attn_with_kvcache(
|
||||
if isinstance(result, tuple):
|
||||
return result[0]
|
||||
return result
|
||||
|
||||
|
||||
@cache
|
||||
def make_image_mask_mod(window_left: int = -1):
|
||||
"""Causal text plus same-image bidirectionality, with a left-window bound.
|
||||
|
||||
aux_tensors are packed query image ranges (inclusive, -1 for text) and
|
||||
cu_seqlens_q. Ranges use absolute positions in each request's KV sequence.
|
||||
Cache the callback so its identity remains stable across forward passes.
|
||||
"""
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cutlass import Int32
|
||||
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.utils import (
|
||||
scalar_to_ssa,
|
||||
ssa_to_scalar,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def image_mask(batch_idx, head_idx, q_idx, kv_idx, seqlen_info, aux_tensors):
|
||||
ranges, cu_q = aux_tensors
|
||||
# Partial tiles can call the mask on padded query rows. Clamp the load;
|
||||
# the kernel separately masks those rows out of the output.
|
||||
local_q = cutlass.min(ssa_to_scalar(q_idx), seqlen_info.seqlen_q - 1)
|
||||
row = cutlass.max(cu_q[batch_idx[0]] + local_q, Int32(0))
|
||||
begin = scalar_to_ssa(ranges[row, 0], Int32)
|
||||
end = scalar_to_ssa(ranges[row, 1], Int32)
|
||||
absolute_q = q_idx + scalar_to_ssa(
|
||||
seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32
|
||||
)
|
||||
keep = (kv_idx <= absolute_q) | ((kv_idx >= begin) & (kv_idx <= end))
|
||||
if cutlass.const_expr(window_left >= 0):
|
||||
keep = keep & (kv_idx >= absolute_q - window_left)
|
||||
return keep
|
||||
|
||||
image_mask.__vec_size__ = 1
|
||||
return image_mask
|
||||
|
||||
@@ -118,13 +118,14 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
)
|
||||
|
||||
def _get_tiled_mma(self):
|
||||
atom_layout_n = 2 if max(self.tile_hdim, self.tile_hdimv) > 256 else 1
|
||||
tiled_mma_qk = sm90_utils_basic.make_trivial_tiled_mma(
|
||||
self.dtype,
|
||||
self.dtype,
|
||||
warpgroup.OperandMajorMode.K,
|
||||
warpgroup.OperandMajorMode.K,
|
||||
Float32,
|
||||
atom_layout_mnk=(self.tile_m // 64, 1, 1),
|
||||
atom_layout_mnk=(self.tile_m // 64, atom_layout_n, 1),
|
||||
tiler_mn=(64, self.tile_n),
|
||||
)
|
||||
tiled_mma_pv = sm90_utils_basic.make_trivial_tiled_mma(
|
||||
@@ -135,10 +136,10 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
Float32,
|
||||
atom_layout_mnk=(
|
||||
self.tile_m // 64,
|
||||
atom_layout_n,
|
||||
1,
|
||||
1,
|
||||
), # Might need (1, 2, 1) for hdim 512
|
||||
tiler_mn=(64, self.tile_hdimv),
|
||||
),
|
||||
tiler_mn=(64, min(256, self.tile_hdimv)),
|
||||
a_source=(
|
||||
warpgroup.OperandSource.RMEM
|
||||
if self.mma_pv_is_rs
|
||||
@@ -1857,10 +1858,23 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
|
||||
if const_expr(not self.mma_pv_is_rs):
|
||||
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
# Both warp groups consume the same P tile for hdim 512.
|
||||
# Wait for every reader before overwriting shared memory.
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PEmpty),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
|
||||
# Fence and barrier to make smem store visible to WGMMA
|
||||
cute.arch.fence_view_async_shared()
|
||||
cute.arch.sync_warp()
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PFull),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
else:
|
||||
cute.arch.sync_warp()
|
||||
|
||||
# For RescaleOBeforeGemm: initialize acc_O
|
||||
if const_expr(self.rescale_O_before_gemm):
|
||||
@@ -1962,12 +1976,25 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
utils.cvt_f16(tOrP_acc, tOrP_cur)
|
||||
if const_expr(not self.mma_pv_is_rs):
|
||||
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
# Both warp groups consume the same P tile for hdim 512.
|
||||
# Wait for every reader before overwriting shared memory.
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PEmpty),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
|
||||
softmax.rescale_O(acc_O, row_scale)
|
||||
if const_expr(not self.mma_pv_is_rs):
|
||||
# Fence and barrier to make sure smem store is visible to WGMMA
|
||||
cute.arch.fence_view_async_shared()
|
||||
cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PFull),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
else:
|
||||
cute.arch.sync_warp()
|
||||
pipeline_v.consumer_wait(
|
||||
smem_pipe_read, pipeline_v.consumer_try_wait(smem_pipe_read)
|
||||
)
|
||||
@@ -2056,6 +2083,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
utils.cvt_f16(tOrP_acc, tOrP_cur)
|
||||
if const_expr(not self.mma_pv_is_rs):
|
||||
tPrP = smem_copy_params.smem_thr_copy_P.retile(tOrP_cur)
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
# Both warp groups consume the same P tile for hdim 512.
|
||||
# Wait for every reader before overwriting shared memory.
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PEmpty),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
cute.copy(smem_copy_params.smem_thr_copy_P, tPrP, smem_copy_params.tPsP)
|
||||
if const_expr(not self.rescale_O_before_gemm):
|
||||
softmax.rescale_O(acc_O, row_scale)
|
||||
@@ -2064,7 +2098,13 @@ class FlashAttentionForwardSm90(FlashAttentionForwardBase):
|
||||
if const_expr(not self.mma_pv_is_rs):
|
||||
# Fence and barrier to make sure smem store is visible to WGMMA
|
||||
cute.arch.fence_view_async_shared()
|
||||
cute.arch.sync_warp() # Only need syncwarp since each warp is using its own P values for MmaPV
|
||||
if const_expr(max(self.tile_hdim, self.tile_hdimv) > 256):
|
||||
cute.arch.barrier(
|
||||
barrier_id=int(NamedBarrierFwd.PFull),
|
||||
number_of_threads=self.num_mma_threads,
|
||||
)
|
||||
else:
|
||||
cute.arch.sync_warp()
|
||||
return smem_pipe_read
|
||||
|
||||
@cute.jit
|
||||
|
||||
@@ -127,13 +127,16 @@ def _validate_head_dims(
|
||||
is_dedicate_kernel_shape = head_dim == 256 and head_dim_v == 256
|
||||
is_standard_range = 8 <= head_dim <= 128 and 8 <= head_dim_v <= 128
|
||||
|
||||
is_sm90_range = 8 <= head_dim <= 256 and 8 <= head_dim_v <= 256
|
||||
is_sm90_range = (8 <= head_dim <= 256 and 8 <= head_dim_v <= 256) or (
|
||||
head_dim == head_dim_v == 512
|
||||
)
|
||||
if compute_capability == 9:
|
||||
assert (
|
||||
is_sm90_range and head_dim % alignment == 0 and head_dim_v % alignment == 0
|
||||
), (
|
||||
f"(head_dim, head_dim_v)=({head_dim}, {head_dim_v}) is not supported on SM90. "
|
||||
f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}."
|
||||
f"head_dim and head_dim_v must be between 8 and 256 and divisible by {alignment}, "
|
||||
"or both equal to 512."
|
||||
)
|
||||
elif compute_capability in [10, 11]:
|
||||
assert (
|
||||
@@ -209,9 +212,11 @@ def _tile_size_fwd_sm90(
|
||||
elif head_dim <= 192:
|
||||
tile_n = 96 if is_local else (128 if head_dim_v <= 128 else 112)
|
||||
return FwdConfig(128, tile_n, True, True)
|
||||
else: # hdim 256
|
||||
elif head_dim <= 256:
|
||||
tile_n = 64 if is_local else 80
|
||||
return FwdConfig(128, tile_n, True, True)
|
||||
else: # hdim 512
|
||||
return FwdConfig(64, 64, False, True)
|
||||
|
||||
|
||||
def maybe_contiguous(x):
|
||||
@@ -295,6 +300,22 @@ def num_splits_heuristic(total_mblocks, num_SMs, num_n_blocks, max_splits):
|
||||
return min(num_SMs // total_mblocks, max_splits, num_n_blocks)
|
||||
|
||||
|
||||
|
||||
def num_splits_heuristic_sm90_decode(total_mblocks, num_SMs, num_n_blocks, max_splits):
|
||||
"""Conservative short-span decode policy; all inputs are host-side bounds.
|
||||
|
||||
A short span (at most 32 KV tiles) with at least a quarter-SM wave of
|
||||
independent query/head blocks amortizes poorly across split/reduce launches.
|
||||
Keep it unsplit. For sparse grids or longer spans retain the upstream
|
||||
occupancy heuristic. These thresholds are a policy, not an autotuned optimum.
|
||||
"""
|
||||
if num_n_blocks <= 32 and total_mblocks * 4 >= num_SMs:
|
||||
return 1
|
||||
return max(1, num_splits_heuristic(
|
||||
total_mblocks, num_SMs, num_n_blocks, max_splits
|
||||
))
|
||||
|
||||
|
||||
def _resolve_causal_local_window(
|
||||
causal, window_size_left, window_size_right, mask_mod=None
|
||||
):
|
||||
@@ -802,8 +823,8 @@ def _flash_attn_fwd(
|
||||
0,
|
||||
min(
|
||||
max_seqlen_k,
|
||||
(window_size_right or max_seqlen_k)
|
||||
+ (window_size_left or max_seqlen_k)
|
||||
(max_seqlen_k if window_size_right is None else window_size_right)
|
||||
+ (max_seqlen_k if window_size_left is None else window_size_left)
|
||||
+ 1
|
||||
+ tile_m,
|
||||
),
|
||||
@@ -860,7 +881,10 @@ def _flash_attn_fwd(
|
||||
)
|
||||
num_splits = arch_forward_plan.num_splits
|
||||
elif num_splits < 1:
|
||||
num_splits = num_splits_heuristic(
|
||||
split_selector = num_splits_heuristic
|
||||
if arch // 10 == 9 and max_seqlen_q == 1:
|
||||
split_selector = num_splits_heuristic_sm90_decode
|
||||
num_splits = split_selector(
|
||||
total_mblocks,
|
||||
num_SMs,
|
||||
num_n_blocks,
|
||||
@@ -1435,8 +1459,7 @@ def _flash_attn_fwd(
|
||||
pack_gqa=pack_gqa,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n,
|
||||
# num_stages=1,
|
||||
num_stages=2,
|
||||
num_stages=1 if max(head_dim, head_dim_v) > 256 else 2,
|
||||
num_threads=num_threads,
|
||||
Q_in_regs=False,
|
||||
intra_wg_overlap=intra_wg_overlap,
|
||||
|
||||
@@ -93,7 +93,8 @@ class PagedKVManager(ParamsBase):
|
||||
atom_async_copy, thr_layout, val_layout
|
||||
)
|
||||
gmem_thr_copy_KV = gmem_tiled_copy_KV.get_slice(thread_idx)
|
||||
page_entry_per_thread = n_block_size // num_threads
|
||||
# Cover short tiles and the remaining rows of a partial final thread group.
|
||||
page_entry_per_thread = (n_block_size + num_threads - 1) // num_threads
|
||||
|
||||
if const_expr(mSFK_paged is not None or mSFV_paged is not None):
|
||||
atom_async_copy_sf = cute.make_copy_atom(
|
||||
@@ -299,6 +300,7 @@ class PagedKVManager(ParamsBase):
|
||||
if n_block >= 0
|
||||
else 0
|
||||
)
|
||||
rows_per_pass = self.num_threads // self.gmem_threads_per_row
|
||||
for m in cutlass.range_constexpr(cute.size(tXsX, mode=[1])):
|
||||
row_valid = tXc0X[0, m, 0][0] < seqlenk_row_limit
|
||||
should_load = cute.make_fragment_like(tXsX[(0, None), m, 0], cute.Boolean)
|
||||
@@ -319,7 +321,14 @@ class PagedKVManager(ParamsBase):
|
||||
mX_paged_cur_copy = cute.tiled_divide(
|
||||
mX_paged_cur, (self.async_copy_elems,)
|
||||
)
|
||||
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
|
||||
if const_expr(tXc0X[0, m, 0][0] + rows_per_pass <= self.n_block_size):
|
||||
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
|
||||
else:
|
||||
# Rows past a partial final pass alias another column block or
|
||||
# stage. Predicated-off cp.async still zero-fills its destination,
|
||||
# so out-of-tile threads must not issue a copy.
|
||||
if tXcX[0, m, 0][0] < self.n_block_size:
|
||||
self._copy_row_async(tXsX, tXcX, mX_paged_cur_copy, m, should_load)
|
||||
|
||||
@cute.jit
|
||||
def load_sf_KV(self, n_block: Int32, sSFX: cute.Tensor, K_or_V: str):
|
||||
|
||||
@@ -643,6 +643,7 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
accepted_backends = (
|
||||
"trtllm_mha",
|
||||
"triton",
|
||||
"fa4",
|
||||
"ascend",
|
||||
"intel_xpu",
|
||||
"intel_amx",
|
||||
@@ -651,7 +652,7 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
assert (
|
||||
prefill_backend in accepted_backends and decode_backend in accepted_backends
|
||||
), (
|
||||
"Gemma4 only supports trtllm_mha, triton, ascend, intel_xpu, intel_amx, or "
|
||||
"Gemma4 only supports trtllm_mha, triton, fa4, ascend, intel_xpu, intel_amx, or "
|
||||
f"aiter attention backend, got prefill={prefill_backend}, decode={decode_backend}"
|
||||
)
|
||||
|
||||
|
||||
@@ -93,6 +93,8 @@ class FlashAttentionMetadata:
|
||||
# CUDA graph (built once per forward, reused across layers). See forward_extend.
|
||||
fa_skip_cu_seqlens_q: torch.Tensor = None
|
||||
fa_skip_max_seqlen_q: int = None
|
||||
# Packed absolute image ranges for Gemma4 prefill; None for text/decode.
|
||||
image_token_ranges: torch.Tensor = None
|
||||
# Window size (typically used by Gemma)
|
||||
window_size: tuple = (-1, -1)
|
||||
# Page table, the index of KV Cache Tables/Blocks
|
||||
@@ -1513,6 +1515,33 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
if score_mod is not None:
|
||||
kwargs["score_mod"] = score_mod
|
||||
kwargs["aux_tensors"] = aux_tensors
|
||||
if metadata.image_token_ranges is not None:
|
||||
if (
|
||||
self.fa_impl_ver != 4
|
||||
or cp_active
|
||||
or use_local_attn
|
||||
or use_cascade_attn
|
||||
or self.use_mla
|
||||
or self.fa_skip_kv_cache
|
||||
or metadata.swa_spec_metadata is not None
|
||||
or metadata is self.full_cg_prefill_metadata
|
||||
or score_mod is not None
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"FA4 image masks require standard paged MHA prefill without "
|
||||
"context parallelism, full prefill graphs, speculative metadata, "
|
||||
"or score modifiers."
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_attention_v4 import (
|
||||
make_image_mask_mod,
|
||||
)
|
||||
|
||||
kwargs["mask_mod"] = make_image_mask_mod(window_size[0])
|
||||
kwargs["aux_tensors"] = [metadata.image_token_ranges, metadata.cu_seqlens_q]
|
||||
# The callback permits future keys within an image. Built-in
|
||||
# causal/window pruning would discard those keys before the mask.
|
||||
causal = False
|
||||
window_size = (-1, -1)
|
||||
kwargs.update(self._mxfp8_sf_kwargs(layer, forward_batch, q_descale))
|
||||
if fa_k_descale is not None:
|
||||
kwargs["k_descale"] = fa_k_descale
|
||||
|
||||
@@ -298,6 +298,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
"""
|
||||
|
||||
_use_draft_input_embeds = False
|
||||
_fa4_prefill = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
_captured_attn_metadata_max_bs: Optional[int] = None
|
||||
dllm_attention = None
|
||||
@@ -316,6 +317,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if self.dllm_attention is not None
|
||||
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
|
||||
)
|
||||
prefill_attn_backend = getattr(
|
||||
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
|
||||
)
|
||||
self._fa4_prefill = getattr(prefill_attn_backend, "fa_impl_ver", None) == 4
|
||||
# --- model flags ----------------------------------------------
|
||||
self.quant_config = getattr(model_runner.model, "quant_config", None)
|
||||
self.is_multimodal = model_runner.model_config.is_multimodal
|
||||
@@ -1288,6 +1293,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
|
||||
):
|
||||
return False
|
||||
# Full replay bypasses forward_extend's Python image-mask guard.
|
||||
# Keep FA4 multimodal prefill eager until its full-graph path is
|
||||
# supported; both forward dispatch and the DP vote use this policy.
|
||||
if self._is_full_backend and self._fa4_prefill and contains_mm_inputs:
|
||||
return False
|
||||
if self._is_full_backend and batch_size > self._capture_req_slots:
|
||||
return False
|
||||
if (
|
||||
|
||||
@@ -335,11 +335,51 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
|
||||
during prefill. Following the HF implementation, bidirectional attention
|
||||
is only enabled within each individual image group (same-item
|
||||
tokens), not across items.
|
||||
Currently only the TritonAttnBackend supports this.
|
||||
Triton and FA4 use causal attention for image spans split across chunks.
|
||||
|
||||
TODO(kpham-sgl): Guard appropriately for gemma3_mm.py:prepare_attn_masks()
|
||||
"""
|
||||
if not isinstance(get_attn_backend(), TritonAttnBackend):
|
||||
attn_backend = get_attn_backend()
|
||||
prefill_backend = getattr(attn_backend, "prefill_backend", attn_backend)
|
||||
if getattr(prefill_backend, "fa_impl_ver", None) == 4:
|
||||
# O(query tokens) metadata instead of a dense Q x KV mask. Keep
|
||||
# absolute ranges so cached text prefixes retain the right offset.
|
||||
lengths = forward_batch.extend_seq_lens_cpu
|
||||
prefixes = forward_batch.extend_prefix_lens_cpu
|
||||
ranges = torch.full((sum(lengths), 2), -1, dtype=torch.int32)
|
||||
offset = 0
|
||||
split_images = []
|
||||
for i, (length, prefix) in enumerate(zip(lengths, prefixes)):
|
||||
mm_inputs = forward_batch.mm_inputs[i]
|
||||
if mm_inputs is not None:
|
||||
for item in mm_inputs.mm_items:
|
||||
if not item.is_image():
|
||||
continue
|
||||
for begin, end in item.offsets:
|
||||
if end < prefix or begin >= prefix + length:
|
||||
continue
|
||||
if begin < prefix or end >= prefix + length:
|
||||
# Match Triton: leave split image rows at the
|
||||
# causal sentinel instead of granting partial
|
||||
# bidirectional attention within this chunk.
|
||||
split_images.append((i, begin, end))
|
||||
continue
|
||||
ranges[
|
||||
offset + begin - prefix : offset + end + 1 - prefix
|
||||
] = torch.tensor([begin, end], dtype=torch.int32)
|
||||
offset += length
|
||||
if split_images:
|
||||
logger.warning_once(
|
||||
f"{len(split_images)} images are split across chunk boundaries. "
|
||||
f"First 5 split images (batch, begin, end): {split_images[:5]}. "
|
||||
"Those images will receive causal attention. Disable chunked "
|
||||
"prefill (--chunked-prefill-size=-1) for full bidirectional attention."
|
||||
)
|
||||
prefill_backend.forward_metadata.image_token_ranges = ranges.to(
|
||||
input_ids.device, non_blocking=True
|
||||
)
|
||||
return
|
||||
if not isinstance(prefill_backend, TritonAttnBackend):
|
||||
logger.warning_once(
|
||||
"Bidirectional attention for image tokens requires TritonAttnBackend. "
|
||||
"Falling back to causal attention, which may degrade image quality."
|
||||
@@ -413,10 +453,10 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
|
||||
)
|
||||
if bidirectional_attn_masks_list:
|
||||
bidirectional_attn_masks = torch.cat(bidirectional_attn_masks_list, dim=0)
|
||||
get_attn_backend().forward_metadata.mask_indptr = (
|
||||
prefill_backend.forward_metadata.mask_indptr = (
|
||||
bidirectional_attn_mask_indptr
|
||||
)
|
||||
get_attn_backend().forward_metadata.custom_mask = bidirectional_attn_masks
|
||||
prefill_backend.forward_metadata.custom_mask = bidirectional_attn_masks
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
vt = self.vision_tower
|
||||
|
||||
Reference in New Issue
Block a user