diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index c4c99ff554..a8761125ab 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -111,6 +111,20 @@ def _dcp_a2a_merge( return merged +def _bidirectional_image_attn_args( + *, + block_ids: Optional[torch.Tensor], + window_size: tuple[int, int], + max_seqlen_k: int, +) -> tuple[Optional[torch.Tensor], tuple[int, int]]: + # The kernel's bidirectional-block mask has no sliding-window variant. Drop the + # window only when it cannot clip a key (kv length <= left window + 1); + # otherwise keep local attention and leave this layer without the mask. + if block_ids is None or (window_size[0] >= 0 and max_seqlen_k > window_size[0] + 1): + return None, window_size + return block_ids, (-1, -1) + + class XPUAttentionBackend(AttentionBackend): """XPU FlashAttention backend, currently based on FlashAttentionBackend, will be refactored later. @@ -217,6 +231,10 @@ class XPUAttentionBackend(AttentionBackend): self.dcp_comm_backend = get_parallel().dcp_comm_backend self._assert_dcp_supported() + # Gemma-3 multimodal: per-token image-block id over whole sequences, set by + # prepare_attn_masks each image EXTEND forward, consumed by forward_extend. + self.bidirectional_block_ids = None + def _assert_dcp_supported(self): if self.has_swa: raise ValueError(_DCP_SWA_MSG) @@ -326,6 +344,7 @@ class XPUAttentionBackend(AttentionBackend): def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize forward metadata hence all layers in the forward pass can reuse it.""" + self.bidirectional_block_ids = None if self.dcp_size > 1: self._init_forward_metadata_dcp(forward_batch) return @@ -846,6 +865,36 @@ class XPUAttentionBackend(AttentionBackend): cu_seqlens_k = metadata.encoder_cu_seqlens_k window_size = (-1, -1) + # Gemma-3 multimodal: prepare_attn_masks set bidirectional_block_ids (per + # token image id over whole sequences, -1 for text). Run non-causal + pass + # it so the sgl-kernel-xpu prefill re-imposes causal for text but keeps + # image tokens bidirectional. + causal_arg = False if use_cascade_attn else causal + bidirectional_block_ids = None + if not layer.is_cross_attention and not use_cascade_attn: + bidirectional_block_ids, window_size = _bidirectional_image_attn_args( + block_ids=self.bidirectional_block_ids, + window_size=window_size, + max_seqlen_k=metadata.max_seq_len_k, + ) + if bidirectional_block_ids is not None: + causal_arg = False + if bidirectional_block_ids is not None and any( + forward_batch.extend_prefix_lens_cpu + ): + o = self._forward_image_blocks_after_prefix( + q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + key_cache=key_cache, + value_cache=value_cache, + page_table=page_table, + forward_batch=forward_batch, + metadata=metadata, + block_ids=bidirectional_block_ids, + scale=layer.scaling, + softcap=layer.logit_cap, + ) + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + result = flash_attn_with_kvcache( q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_cache=key_cache, @@ -856,9 +905,10 @@ class XPUAttentionBackend(AttentionBackend): cu_seqlens_k_new=None, max_seqlen_q=max_seqlen_q, softmax_scale=layer.scaling, - causal=False if use_cascade_attn else causal, + causal=causal_arg, window_size=window_size, softcap=layer.logit_cap, + bidirectional_block_ids=bidirectional_block_ids, k_descale=k_descale, v_descale=v_descale, return_softmax_lse=use_cascade_attn, @@ -1066,6 +1116,47 @@ class XPUAttentionBackend(AttentionBackend): return_softmax_lse=return_softmax_lse, ) + def _forward_image_blocks_after_prefix( + self, + *, + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + page_table: torch.Tensor, + forward_batch: ForwardBatch, + metadata: FlashAttentionMetadata, + block_ids: torch.Tensor, + scale: float, + softcap: float, + ) -> torch.Tensor: + # The kernel's block mask needs each query length to equal its KV length, so + # pad the queries to whole sequences with placeholders over the cached prefix. + num_kv_tokens = q.shape[0] + sum(forward_batch.extend_prefix_lens_cpu) + # Extend token t of sequence i sits at row t + sum(prefix_lens[: i + 1]). + prefix_offsets = torch.repeat_interleave( + torch.cumsum(forward_batch.extend_prefix_lens, dim=0), + forward_batch.extend_seq_lens, + output_size=q.shape[0], + ) + extend_rows = torch.arange(q.shape[0], device=q.device) + prefix_offsets + padded_q = q.new_zeros((num_kv_tokens, *q.shape[1:])) + padded_q[extend_rows] = q + o = flash_attn_with_kvcache( + q=padded_q, + k_cache=key_cache, + v_cache=value_cache, + page_table=page_table, + cache_seqlens=metadata.cache_seqlens_int32, + cu_seqlens_q=metadata.cu_seqlens_k, + max_seqlen_q=metadata.max_seq_len_k, + softmax_scale=scale, + causal=False, + softcap=softcap, + bidirectional_block_ids=block_ids, + num_splits=self.num_splits, + ) + return o[extend_rows] + def _forward_encoder_decoder_attn( self, q: torch.Tensor, diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 6068e8f209..4ee8c1e5f7 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1352,9 +1352,13 @@ class Gemma3RMSNorm(BaseFusedOp): # output and residual becomes x + residual for the next layer. gemma_fused_add_rmsnorm(x, residual, self.weight.data, self.eps) return x, residual - # The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs - # can all go through it directly without a Python-side reshape. + # The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs go + # through directly without a Python-side reshape. It handles + # non-contiguous batch/row strides but requires a contiguous last dim, + # which a permuted hidden state may not have. elif residual is None and x.dim() in (2, 3, 4): + if x.stride(-1) != 1: + x = x.contiguous() return gemma_rmsnorm(x, self.weight.data, self.eps) return self.forward_native(x, residual) diff --git a/python/sglang/srt/models/gemma3_mm.py b/python/sglang/srt/models/gemma3_mm.py index e6eb155527..e8bad25e20 100644 --- a/python/sglang/srt/models/gemma3_mm.py +++ b/python/sglang/srt/models/gemma3_mm.py @@ -48,7 +48,7 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.gemma3_causal import Gemma3ForCausalLM from sglang.srt.models.siglip import SiglipVisionModel -from sglang.srt.utils import add_prefix +from sglang.srt.utils import add_prefix, is_xpu from sglang.srt.utils.hf_transformers_utils import get_processor logger = logging.getLogger(__name__) @@ -220,7 +220,8 @@ class Gemma3ForConditionalGeneration(PreTrainedModel): mask_dtype: torch.dtype, ): """Prepare attention masks for multimodal inputs.""" - if isinstance(get_attn_backend(), TritonAttnBackend): + backend = get_attn_backend() + if isinstance(backend, TritonAttnBackend): assert forward_batch.forward_mode == ForwardMode.EXTEND bidirectional_attn_masks_list = [] bidirectional_attn_mask_indptr = torch.zeros( @@ -271,6 +272,47 @@ class Gemma3ForConditionalGeneration(PreTrainedModel): get_attn_backend().forward_metadata.custom_mask = ( bidirectional_attn_masks ) + elif is_xpu(): + # XPU takes a per extend-token image-block id (image index, -1 for + # text); the sgl-kernel-xpu FMHA prefill turns it into bidirectional + # image attention (see xpu_backend.forward_extend). Import lazily so + # non-XPU builds are unaffected. + from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend + + if isinstance(backend, XPUAttentionBackend): + assert forward_batch.forward_mode == ForwardMode.EXTEND + backend.bidirectional_block_ids = self._build_xpu_image_block_ids( + forward_batch=forward_batch, device=input_ids.device + ) + + @staticmethod + def _build_xpu_image_block_ids(forward_batch: ForwardBatch, device): + # Image spans are sglang layout, so they are read here; the kernel package + # cannot import sglang types and only builds the id tensor. + from sgl_kernel.flash_attn import build_bidirectional_block_ids + + # Ids cover whole sequences (cached prefix + extend); with a prefix the backend + # pads the queries to match. None keeps the plain causal path. + seq_lens, block_spans = [], [] + for i in range(forward_batch.batch_size): + start = forward_batch.extend_prefix_lens_cpu[i] + end = start + forward_batch.extend_seq_lens_cpu[i] + spans = [] + mm_inputs = forward_batch.mm_inputs[i] + if mm_inputs is not None: + for mm_item in mm_inputs.mm_items: + if mm_item.is_image(): + for begin, last in mm_item.offsets: + last = min(last, end - 1) + # Fewer than two image tokens in this extend attend the + # same keys under causal attention. + if max(begin, start) < last: + spans.append((begin, last)) + seq_lens.append(end) + block_spans.append(spans) + return build_bidirectional_block_ids( + seq_lens=seq_lens, block_spans=block_spans, device=device + ) def get_input_embeddings(self) -> nn.Embedding: return self.language_model.get_input_embeddings() diff --git a/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py b/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py new file mode 100644 index 0000000000..005715d2f8 --- /dev/null +++ b/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py @@ -0,0 +1,222 @@ +"""Gemma-3 bidirectional image attention on XPU (sgl-kernel-xpu bidirectional_block_ids). + +Guards the cases the kernel mask cannot express directly: a sliding-window layer +whose prompt exceeds the window must keep local attention, and image tokens in an +extend after a cached prefix must still attend bidirectionally. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase + +register_xpu_ci(est_time=30, suite="stage-b-test-1-gpu-xpu") + +PAGE, H, HKV, D = 128, 8, 4, 128 + + +def _paged_inputs(seq_len, dev): + q = torch.randn(seq_len, H, D, device=dev, dtype=torch.bfloat16) + k = torch.randn(seq_len, HKV, D, device=dev, dtype=torch.bfloat16) + v = torch.randn(seq_len, HKV, D, device=dev, dtype=torch.bfloat16) + npages = (seq_len + PAGE - 1) // PAGE + k_cache = torch.zeros(npages * PAGE, HKV, D, device=dev, dtype=torch.bfloat16) + v_cache = torch.zeros_like(k_cache) + k_cache[:seq_len], v_cache[:seq_len] = k, v + return ( + q, + k, + v, + k_cache.view(npages, PAGE, HKV, D), + v_cache.view(npages, PAGE, HKV, D), + ) + + +def _reference(q, k, v, allow, scale): + k = k.float().repeat_interleave(H // HKV, dim=1) + v = v.float().repeat_interleave(H // HKV, dim=1) + scores = torch.einsum("qhd,khd->hqk", q.float(), k) * scale + scores = scores.masked_fill(~allow.unsqueeze(0), float("-inf")) + return torch.einsum("hqk,khd->qhd", scores.softmax(dim=-1), v) + + +def _image_batch(prefix_lens, extend_lens, image_spans): + image_items = [ + SimpleNamespace(mm_items=[SimpleNamespace(is_image=lambda: True, offsets=s)]) + for s in image_spans + ] + return SimpleNamespace( + batch_size=len(prefix_lens), + extend_prefix_lens_cpu=prefix_lens, + extend_seq_lens_cpu=extend_lens, + mm_inputs=image_items, + ) + + +def _build_block_ids(forward_batch, device): + from sglang.srt.models.gemma3_mm import Gemma3ForConditionalGeneration + + return Gemma3ForConditionalGeneration._build_xpu_image_block_ids( + forward_batch=forward_batch, device=device + ) + + +@unittest.skipUnless( + hasattr(torch, "xpu") and torch.xpu.is_available(), "requires an Intel XPU" +) +class TestXPUGemma3SlidingWindowBound(CustomTestCase): + WINDOW_LEFT = 63 + SPAN = (8, 40) + + def _run(self, seq_len): + from sgl_kernel.flash_attn import ( + build_bidirectional_block_ids, + flash_attn_with_kvcache, + ) + + from sglang.srt.layers.attention.xpu_backend import ( + _bidirectional_image_attn_args, + ) + + torch.manual_seed(0) + dev = "xpu" + scale = D**-0.5 + q, k, v, k_cache, v_cache = _paged_inputs(seq_len, dev) + block_ids, window_size = _bidirectional_image_attn_args( + block_ids=build_bidirectional_block_ids([seq_len], [[self.SPAN]], dev), + window_size=(self.WINDOW_LEFT, 0), + max_seqlen_k=seq_len, + ) + out = flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + page_table=torch.arange( + k_cache.shape[0], device=dev, dtype=torch.int32 + ).view(1, -1), + cache_seqlens=torch.tensor([seq_len], device=dev, dtype=torch.int32), + cu_seqlens_q=torch.tensor([0, seq_len], device=dev, dtype=torch.int32), + max_seqlen_q=seq_len, + softmax_scale=scale, + causal=block_ids is None, + window_size=window_size, + bidirectional_block_ids=block_ids, + ) + torch.xpu.synchronize() + rows = torch.arange(seq_len, device=dev).view(-1, 1) + cols = torch.arange(seq_len, device=dev).view(1, -1) + return out.float(), q, k, v, rows, cols, scale + + def test_prompt_longer_than_window_keeps_local_attention(self): + """Keys beyond the sliding window must stay masked once the prompt outgrows + the window, even though the batch carries image block ids.""" + out, q, k, v, rows, cols, scale = self._run(seq_len=200) + local = (cols <= rows) & (rows - cols <= self.WINDOW_LEFT) + want = _reference(q, k, v, local, scale) + torch.testing.assert_close(out, want, rtol=2e-2, atol=2e-2) + + def test_prompt_within_window_keeps_bidirectional_image(self): + """At kv length == left window + 1 the window clips nothing, so the image + span must still be attended bidirectionally.""" + out, q, k, v, rows, cols, scale = self._run(seq_len=self.WINDOW_LEFT + 1) + allow = cols <= rows + begin, last = self.SPAN + allow[begin : last + 1, begin : last + 1] = True + want = _reference(q, k, v, allow, scale) + torch.testing.assert_close(out, want, rtol=2e-2, atol=2e-2) + + +@unittest.skipUnless( + hasattr(torch, "xpu") and torch.xpu.is_available(), "requires an Intel XPU" +) +class TestXPUGemma3ImageAfterCachedPrefix(CustomTestCase): + def test_matches_full_bidirectional_attention(self): + """With a cached prefix (radix hit or an earlier chunk), image tokens in the + extend must attend bidirectionally and see the whole prefix. The batch mixes + a page-aligned prefix, an unaligned prefix with an image straddling it, and + no prefix with an image cut by the chunk end.""" + from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend + + torch.manual_seed(0) + dev = "xpu" + scale = D**-0.5 + prefix_lens, extend_lens = [128, 50, 0], [100, 90, 70] + image_spans = [[(140, 190)], [(30, 100)], [(40, 90)]] + seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] + pages_per_seq = [(s + PAGE - 1) // PAGE for s in seq_lens] + num_pages = sum(pages_per_seq) + + # Hand pages out in reverse so each sequence's pages are non-contiguous. + free_pages = list(range(num_pages))[::-1] + req_to_token = torch.zeros(len(seq_lens), 256, dtype=torch.int32, device=dev) + for i, n in enumerate(pages_per_seq): + pages = torch.tensor([free_pages.pop(0) for _ in range(n)], device=dev) + slots = (pages[:, None] * PAGE + torch.arange(PAGE, device=dev)).flatten() + req_to_token[i, : n * PAGE] = slots.to(torch.int32) + k_flat = torch.randn(num_pages * PAGE, HKV, D, device=dev, dtype=torch.bfloat16) + v_flat = torch.randn_like(k_flat) + q = torch.randn(sum(extend_lens), H, D, device=dev, dtype=torch.bfloat16) + + backend = XPUAttentionBackend.__new__(XPUAttentionBackend) + backend.num_splits = 0 + to_dev = lambda x: torch.tensor(x, device=dev, dtype=torch.int32) + block_ids = _build_block_ids( + _image_batch(prefix_lens, extend_lens, image_spans), dev + ) + out = backend._forward_image_blocks_after_prefix( + q=q, + key_cache=k_flat.view(num_pages, PAGE, HKV, D), + value_cache=v_flat.view(num_pages, PAGE, HKV, D), + page_table=req_to_token[:, ::PAGE] // PAGE, + forward_batch=SimpleNamespace( + extend_prefix_lens_cpu=prefix_lens, + extend_prefix_lens=to_dev(prefix_lens), + extend_seq_lens=to_dev(extend_lens), + ), + metadata=SimpleNamespace( + cache_seqlens_int32=to_dev(seq_lens), + cu_seqlens_k=torch.nn.functional.pad( + torch.cumsum(to_dev(seq_lens), dim=0, dtype=torch.int32), (1, 0) + ), + max_seq_len_k=max(seq_lens), + ), + block_ids=block_ids, + scale=scale, + softcap=0.0, + ) + torch.xpu.synchronize() + + q_start = 0 + for i, (prefix_len, seq_len) in enumerate(zip(prefix_lens, seq_lens)): + slots = req_to_token[i, :seq_len].long() + rows = torch.arange(prefix_len, seq_len, device=dev).view(-1, 1) + cols = torch.arange(seq_len, device=dev).view(1, -1) + allow = cols <= rows + for begin, last in image_spans[i]: + allow |= ( + (rows >= begin) & (rows <= last) & (cols >= begin) & (cols <= last) + ) + q_end = q_start + seq_len - prefix_len + want = _reference( + q[q_start:q_end], k_flat[slots], v_flat[slots], allow, scale + ) + torch.testing.assert_close( + out[q_start:q_end].float(), want, rtol=2e-2, atol=2e-2 + ) + q_start = q_end + + +class TestXPUGemma3ImageBlockIds(CustomTestCase): + def test_image_only_in_cached_prefix_needs_no_mask(self): + """A text-only extend after a cached image is exact under causal attention + and must take the plain causal path.""" + self.assertIsNone( + _build_block_ids(_image_batch([256], [50], [[(10, 200)]]), "cpu") + ) + + +if __name__ == "__main__": + unittest.main()