[Fix] [XPU] gemma3: Enable bidirectional image attention (#44225)

This commit is contained in:
desmond-intel
2026-10-19 14:08:24 +08:00
committed by GitHub
parent a3330237db
commit dc81f17b90
4 changed files with 364 additions and 5 deletions
@@ -111,6 +111,20 @@ def _dcp_a2a_merge(
return merged
def _bidirectional_image_attn_args(
*,
block_ids: Optional[torch.Tensor],
window_size: tuple[int, int],
max_seqlen_k: int,
) -> tuple[Optional[torch.Tensor], tuple[int, int]]:
# The kernel's bidirectional-block mask has no sliding-window variant. Drop the
# window only when it cannot clip a key (kv length <= left window + 1);
# otherwise keep local attention and leave this layer without the mask.
if block_ids is None or (window_size[0] >= 0 and max_seqlen_k > window_size[0] + 1):
return None, window_size
return block_ids, (-1, -1)
class XPUAttentionBackend(AttentionBackend):
"""XPU FlashAttention backend, currently based on FlashAttentionBackend, will be refactored later.
@@ -217,6 +231,10 @@ class XPUAttentionBackend(AttentionBackend):
self.dcp_comm_backend = get_parallel().dcp_comm_backend
self._assert_dcp_supported()
# Gemma-3 multimodal: per-token image-block id over whole sequences, set by
# prepare_attn_masks each image EXTEND forward, consumed by forward_extend.
self.bidirectional_block_ids = None
def _assert_dcp_supported(self):
if self.has_swa:
raise ValueError(_DCP_SWA_MSG)
@@ -326,6 +344,7 @@ class XPUAttentionBackend(AttentionBackend):
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Initialize forward metadata hence all layers in the forward pass can reuse it."""
self.bidirectional_block_ids = None
if self.dcp_size > 1:
self._init_forward_metadata_dcp(forward_batch)
return
@@ -846,6 +865,36 @@ class XPUAttentionBackend(AttentionBackend):
cu_seqlens_k = metadata.encoder_cu_seqlens_k
window_size = (-1, -1)
# Gemma-3 multimodal: prepare_attn_masks set bidirectional_block_ids (per
# token image id over whole sequences, -1 for text). Run non-causal + pass
# it so the sgl-kernel-xpu prefill re-imposes causal for text but keeps
# image tokens bidirectional.
causal_arg = False if use_cascade_attn else causal
bidirectional_block_ids = None
if not layer.is_cross_attention and not use_cascade_attn:
bidirectional_block_ids, window_size = _bidirectional_image_attn_args(
block_ids=self.bidirectional_block_ids,
window_size=window_size,
max_seqlen_k=metadata.max_seq_len_k,
)
if bidirectional_block_ids is not None:
causal_arg = False
if bidirectional_block_ids is not None and any(
forward_batch.extend_prefix_lens_cpu
):
o = self._forward_image_blocks_after_prefix(
q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
key_cache=key_cache,
value_cache=value_cache,
page_table=page_table,
forward_batch=forward_batch,
metadata=metadata,
block_ids=bidirectional_block_ids,
scale=layer.scaling,
softcap=layer.logit_cap,
)
return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
result = flash_attn_with_kvcache(
q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_cache=key_cache,
@@ -856,9 +905,10 @@ class XPUAttentionBackend(AttentionBackend):
cu_seqlens_k_new=None,
max_seqlen_q=max_seqlen_q,
softmax_scale=layer.scaling,
causal=False if use_cascade_attn else causal,
causal=causal_arg,
window_size=window_size,
softcap=layer.logit_cap,
bidirectional_block_ids=bidirectional_block_ids,
k_descale=k_descale,
v_descale=v_descale,
return_softmax_lse=use_cascade_attn,
@@ -1066,6 +1116,47 @@ class XPUAttentionBackend(AttentionBackend):
return_softmax_lse=return_softmax_lse,
)
def _forward_image_blocks_after_prefix(
self,
*,
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
page_table: torch.Tensor,
forward_batch: ForwardBatch,
metadata: FlashAttentionMetadata,
block_ids: torch.Tensor,
scale: float,
softcap: float,
) -> torch.Tensor:
# The kernel's block mask needs each query length to equal its KV length, so
# pad the queries to whole sequences with placeholders over the cached prefix.
num_kv_tokens = q.shape[0] + sum(forward_batch.extend_prefix_lens_cpu)
# Extend token t of sequence i sits at row t + sum(prefix_lens[: i + 1]).
prefix_offsets = torch.repeat_interleave(
torch.cumsum(forward_batch.extend_prefix_lens, dim=0),
forward_batch.extend_seq_lens,
output_size=q.shape[0],
)
extend_rows = torch.arange(q.shape[0], device=q.device) + prefix_offsets
padded_q = q.new_zeros((num_kv_tokens, *q.shape[1:]))
padded_q[extend_rows] = q
o = flash_attn_with_kvcache(
q=padded_q,
k_cache=key_cache,
v_cache=value_cache,
page_table=page_table,
cache_seqlens=metadata.cache_seqlens_int32,
cu_seqlens_q=metadata.cu_seqlens_k,
max_seqlen_q=metadata.max_seq_len_k,
softmax_scale=scale,
causal=False,
softcap=softcap,
bidirectional_block_ids=block_ids,
num_splits=self.num_splits,
)
return o[extend_rows]
def _forward_encoder_decoder_attn(
self,
q: torch.Tensor,
+6 -2
View File
@@ -1352,9 +1352,13 @@ class Gemma3RMSNorm(BaseFusedOp):
# output and residual becomes x + residual for the next layer.
gemma_fused_add_rmsnorm(x, residual, self.weight.data, self.eps)
return x, residual
# The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs
# can all go through it directly without a Python-side reshape.
# The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs go
# through directly without a Python-side reshape. It handles
# non-contiguous batch/row strides but requires a contiguous last dim,
# which a permuted hidden state may not have.
elif residual is None and x.dim() in (2, 3, 4):
if x.stride(-1) != 1:
x = x.contiguous()
return gemma_rmsnorm(x, self.weight.data, self.eps)
return self.forward_native(x, residual)
+44 -2
View File
@@ -48,7 +48,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.gemma3_causal import Gemma3ForCausalLM
from sglang.srt.models.siglip import SiglipVisionModel
from sglang.srt.utils import add_prefix
from sglang.srt.utils import add_prefix, is_xpu
from sglang.srt.utils.hf_transformers_utils import get_processor
logger = logging.getLogger(__name__)
@@ -221,7 +221,8 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
mask_dtype: torch.dtype,
):
"""Prepare attention masks for multimodal inputs."""
if isinstance(get_attn_backend(), TritonAttnBackend):
backend = get_attn_backend()
if isinstance(backend, TritonAttnBackend):
assert forward_batch.forward_mode == ForwardMode.EXTEND
bidirectional_attn_masks_list = []
bidirectional_attn_mask_indptr = torch.zeros(
@@ -272,6 +273,47 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
get_attn_backend().forward_metadata.custom_mask = (
bidirectional_attn_masks
)
elif is_xpu():
# XPU takes a per extend-token image-block id (image index, -1 for
# text); the sgl-kernel-xpu FMHA prefill turns it into bidirectional
# image attention (see xpu_backend.forward_extend). Import lazily so
# non-XPU builds are unaffected.
from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend
if isinstance(backend, XPUAttentionBackend):
assert forward_batch.forward_mode == ForwardMode.EXTEND
backend.bidirectional_block_ids = self._build_xpu_image_block_ids(
forward_batch=forward_batch, device=input_ids.device
)
@staticmethod
def _build_xpu_image_block_ids(forward_batch: ForwardBatch, device):
# Image spans are sglang layout, so they are read here; the kernel package
# cannot import sglang types and only builds the id tensor.
from sgl_kernel.flash_attn import build_bidirectional_block_ids
# Ids cover whole sequences (cached prefix + extend); with a prefix the backend
# pads the queries to match. None keeps the plain causal path.
seq_lens, block_spans = [], []
for i in range(forward_batch.batch_size):
start = forward_batch.extend_prefix_lens_cpu[i]
end = start + forward_batch.extend_seq_lens_cpu[i]
spans = []
mm_inputs = forward_batch.mm_inputs[i]
if mm_inputs is not None:
for mm_item in mm_inputs.mm_items:
if mm_item.is_image():
for begin, last in mm_item.offsets:
last = min(last, end - 1)
# Fewer than two image tokens in this extend attend the
# same keys under causal attention.
if max(begin, start) < last:
spans.append((begin, last))
seq_lens.append(end)
block_spans.append(spans)
return build_bidirectional_block_ids(
seq_lens=seq_lens, block_spans=block_spans, device=device
)
def get_input_embeddings(self) -> nn.Embedding:
return self.language_model.get_input_embeddings()