diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index a8761125ab..c4c99ff554 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -111,20 +111,6 @@ def _dcp_a2a_merge( return merged -def _bidirectional_image_attn_args( - *, - block_ids: Optional[torch.Tensor], - window_size: tuple[int, int], - max_seqlen_k: int, -) -> tuple[Optional[torch.Tensor], tuple[int, int]]: - # The kernel's bidirectional-block mask has no sliding-window variant. Drop the - # window only when it cannot clip a key (kv length <= left window + 1); - # otherwise keep local attention and leave this layer without the mask. - if block_ids is None or (window_size[0] >= 0 and max_seqlen_k > window_size[0] + 1): - return None, window_size - return block_ids, (-1, -1) - - class XPUAttentionBackend(AttentionBackend): """XPU FlashAttention backend, currently based on FlashAttentionBackend, will be refactored later. @@ -231,10 +217,6 @@ class XPUAttentionBackend(AttentionBackend): self.dcp_comm_backend = get_parallel().dcp_comm_backend self._assert_dcp_supported() - # Gemma-3 multimodal: per-token image-block id over whole sequences, set by - # prepare_attn_masks each image EXTEND forward, consumed by forward_extend. - self.bidirectional_block_ids = None - def _assert_dcp_supported(self): if self.has_swa: raise ValueError(_DCP_SWA_MSG) @@ -344,7 +326,6 @@ class XPUAttentionBackend(AttentionBackend): def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize forward metadata hence all layers in the forward pass can reuse it.""" - self.bidirectional_block_ids = None if self.dcp_size > 1: self._init_forward_metadata_dcp(forward_batch) return @@ -865,36 +846,6 @@ class XPUAttentionBackend(AttentionBackend): cu_seqlens_k = metadata.encoder_cu_seqlens_k window_size = (-1, -1) - # Gemma-3 multimodal: prepare_attn_masks set bidirectional_block_ids (per - # token image id over whole sequences, -1 for text). Run non-causal + pass - # it so the sgl-kernel-xpu prefill re-imposes causal for text but keeps - # image tokens bidirectional. - causal_arg = False if use_cascade_attn else causal - bidirectional_block_ids = None - if not layer.is_cross_attention and not use_cascade_attn: - bidirectional_block_ids, window_size = _bidirectional_image_attn_args( - block_ids=self.bidirectional_block_ids, - window_size=window_size, - max_seqlen_k=metadata.max_seq_len_k, - ) - if bidirectional_block_ids is not None: - causal_arg = False - if bidirectional_block_ids is not None and any( - forward_batch.extend_prefix_lens_cpu - ): - o = self._forward_image_blocks_after_prefix( - q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), - key_cache=key_cache, - value_cache=value_cache, - page_table=page_table, - forward_batch=forward_batch, - metadata=metadata, - block_ids=bidirectional_block_ids, - scale=layer.scaling, - softcap=layer.logit_cap, - ) - return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) - result = flash_attn_with_kvcache( q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_cache=key_cache, @@ -905,10 +856,9 @@ class XPUAttentionBackend(AttentionBackend): cu_seqlens_k_new=None, max_seqlen_q=max_seqlen_q, softmax_scale=layer.scaling, - causal=causal_arg, + causal=False if use_cascade_attn else causal, window_size=window_size, softcap=layer.logit_cap, - bidirectional_block_ids=bidirectional_block_ids, k_descale=k_descale, v_descale=v_descale, return_softmax_lse=use_cascade_attn, @@ -1116,47 +1066,6 @@ class XPUAttentionBackend(AttentionBackend): return_softmax_lse=return_softmax_lse, ) - def _forward_image_blocks_after_prefix( - self, - *, - q: torch.Tensor, - key_cache: torch.Tensor, - value_cache: torch.Tensor, - page_table: torch.Tensor, - forward_batch: ForwardBatch, - metadata: FlashAttentionMetadata, - block_ids: torch.Tensor, - scale: float, - softcap: float, - ) -> torch.Tensor: - # The kernel's block mask needs each query length to equal its KV length, so - # pad the queries to whole sequences with placeholders over the cached prefix. - num_kv_tokens = q.shape[0] + sum(forward_batch.extend_prefix_lens_cpu) - # Extend token t of sequence i sits at row t + sum(prefix_lens[: i + 1]). - prefix_offsets = torch.repeat_interleave( - torch.cumsum(forward_batch.extend_prefix_lens, dim=0), - forward_batch.extend_seq_lens, - output_size=q.shape[0], - ) - extend_rows = torch.arange(q.shape[0], device=q.device) + prefix_offsets - padded_q = q.new_zeros((num_kv_tokens, *q.shape[1:])) - padded_q[extend_rows] = q - o = flash_attn_with_kvcache( - q=padded_q, - k_cache=key_cache, - v_cache=value_cache, - page_table=page_table, - cache_seqlens=metadata.cache_seqlens_int32, - cu_seqlens_q=metadata.cu_seqlens_k, - max_seqlen_q=metadata.max_seq_len_k, - softmax_scale=scale, - causal=False, - softcap=softcap, - bidirectional_block_ids=block_ids, - num_splits=self.num_splits, - ) - return o[extend_rows] - def _forward_encoder_decoder_attn( self, q: torch.Tensor, diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 4ee8c1e5f7..6068e8f209 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1352,13 +1352,9 @@ class Gemma3RMSNorm(BaseFusedOp): # output and residual becomes x + residual for the next layer. gemma_fused_add_rmsnorm(x, residual, self.weight.data, self.eps) return x, residual - # The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs go - # through directly without a Python-side reshape. It handles - # non-contiguous batch/row strides but requires a contiguous last dim, - # which a permuted hidden state may not have. + # The XPU kernel flattens leading dims internally, so 2D/3D/4D inputs + # can all go through it directly without a Python-side reshape. elif residual is None and x.dim() in (2, 3, 4): - if x.stride(-1) != 1: - x = x.contiguous() return gemma_rmsnorm(x, self.weight.data, self.eps) return self.forward_native(x, residual) diff --git a/python/sglang/srt/models/gemma3_mm.py b/python/sglang/srt/models/gemma3_mm.py index e8bad25e20..e6eb155527 100644 --- a/python/sglang/srt/models/gemma3_mm.py +++ b/python/sglang/srt/models/gemma3_mm.py @@ -48,7 +48,7 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.gemma3_causal import Gemma3ForCausalLM from sglang.srt.models.siglip import SiglipVisionModel -from sglang.srt.utils import add_prefix, is_xpu +from sglang.srt.utils import add_prefix from sglang.srt.utils.hf_transformers_utils import get_processor logger = logging.getLogger(__name__) @@ -220,8 +220,7 @@ class Gemma3ForConditionalGeneration(PreTrainedModel): mask_dtype: torch.dtype, ): """Prepare attention masks for multimodal inputs.""" - backend = get_attn_backend() - if isinstance(backend, TritonAttnBackend): + if isinstance(get_attn_backend(), TritonAttnBackend): assert forward_batch.forward_mode == ForwardMode.EXTEND bidirectional_attn_masks_list = [] bidirectional_attn_mask_indptr = torch.zeros( @@ -272,47 +271,6 @@ class Gemma3ForConditionalGeneration(PreTrainedModel): get_attn_backend().forward_metadata.custom_mask = ( bidirectional_attn_masks ) - elif is_xpu(): - # XPU takes a per extend-token image-block id (image index, -1 for - # text); the sgl-kernel-xpu FMHA prefill turns it into bidirectional - # image attention (see xpu_backend.forward_extend). Import lazily so - # non-XPU builds are unaffected. - from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend - - if isinstance(backend, XPUAttentionBackend): - assert forward_batch.forward_mode == ForwardMode.EXTEND - backend.bidirectional_block_ids = self._build_xpu_image_block_ids( - forward_batch=forward_batch, device=input_ids.device - ) - - @staticmethod - def _build_xpu_image_block_ids(forward_batch: ForwardBatch, device): - # Image spans are sglang layout, so they are read here; the kernel package - # cannot import sglang types and only builds the id tensor. - from sgl_kernel.flash_attn import build_bidirectional_block_ids - - # Ids cover whole sequences (cached prefix + extend); with a prefix the backend - # pads the queries to match. None keeps the plain causal path. - seq_lens, block_spans = [], [] - for i in range(forward_batch.batch_size): - start = forward_batch.extend_prefix_lens_cpu[i] - end = start + forward_batch.extend_seq_lens_cpu[i] - spans = [] - mm_inputs = forward_batch.mm_inputs[i] - if mm_inputs is not None: - for mm_item in mm_inputs.mm_items: - if mm_item.is_image(): - for begin, last in mm_item.offsets: - last = min(last, end - 1) - # Fewer than two image tokens in this extend attend the - # same keys under causal attention. - if max(begin, start) < last: - spans.append((begin, last)) - seq_lens.append(end) - block_spans.append(spans) - return build_bidirectional_block_ids( - seq_lens=seq_lens, block_spans=block_spans, device=device - ) def get_input_embeddings(self) -> nn.Embedding: return self.language_model.get_input_embeddings() diff --git a/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py b/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py deleted file mode 100644 index 005715d2f8..0000000000 --- a/test/registered/xpu/test_xpu_gemma3_bidirectional_attn.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Gemma-3 bidirectional image attention on XPU (sgl-kernel-xpu bidirectional_block_ids). - -Guards the cases the kernel mask cannot express directly: a sliding-window layer -whose prompt exceeds the window must keep local attention, and image tokens in an -extend after a cached prefix must still attend bidirectionally. -""" - -import unittest -from types import SimpleNamespace - -import torch - -from sglang.test.ci.ci_register import register_xpu_ci -from sglang.test.test_utils import CustomTestCase - -register_xpu_ci(est_time=30, suite="stage-b-test-1-gpu-xpu") - -PAGE, H, HKV, D = 128, 8, 4, 128 - - -def _paged_inputs(seq_len, dev): - q = torch.randn(seq_len, H, D, device=dev, dtype=torch.bfloat16) - k = torch.randn(seq_len, HKV, D, device=dev, dtype=torch.bfloat16) - v = torch.randn(seq_len, HKV, D, device=dev, dtype=torch.bfloat16) - npages = (seq_len + PAGE - 1) // PAGE - k_cache = torch.zeros(npages * PAGE, HKV, D, device=dev, dtype=torch.bfloat16) - v_cache = torch.zeros_like(k_cache) - k_cache[:seq_len], v_cache[:seq_len] = k, v - return ( - q, - k, - v, - k_cache.view(npages, PAGE, HKV, D), - v_cache.view(npages, PAGE, HKV, D), - ) - - -def _reference(q, k, v, allow, scale): - k = k.float().repeat_interleave(H // HKV, dim=1) - v = v.float().repeat_interleave(H // HKV, dim=1) - scores = torch.einsum("qhd,khd->hqk", q.float(), k) * scale - scores = scores.masked_fill(~allow.unsqueeze(0), float("-inf")) - return torch.einsum("hqk,khd->qhd", scores.softmax(dim=-1), v) - - -def _image_batch(prefix_lens, extend_lens, image_spans): - image_items = [ - SimpleNamespace(mm_items=[SimpleNamespace(is_image=lambda: True, offsets=s)]) - for s in image_spans - ] - return SimpleNamespace( - batch_size=len(prefix_lens), - extend_prefix_lens_cpu=prefix_lens, - extend_seq_lens_cpu=extend_lens, - mm_inputs=image_items, - ) - - -def _build_block_ids(forward_batch, device): - from sglang.srt.models.gemma3_mm import Gemma3ForConditionalGeneration - - return Gemma3ForConditionalGeneration._build_xpu_image_block_ids( - forward_batch=forward_batch, device=device - ) - - -@unittest.skipUnless( - hasattr(torch, "xpu") and torch.xpu.is_available(), "requires an Intel XPU" -) -class TestXPUGemma3SlidingWindowBound(CustomTestCase): - WINDOW_LEFT = 63 - SPAN = (8, 40) - - def _run(self, seq_len): - from sgl_kernel.flash_attn import ( - build_bidirectional_block_ids, - flash_attn_with_kvcache, - ) - - from sglang.srt.layers.attention.xpu_backend import ( - _bidirectional_image_attn_args, - ) - - torch.manual_seed(0) - dev = "xpu" - scale = D**-0.5 - q, k, v, k_cache, v_cache = _paged_inputs(seq_len, dev) - block_ids, window_size = _bidirectional_image_attn_args( - block_ids=build_bidirectional_block_ids([seq_len], [[self.SPAN]], dev), - window_size=(self.WINDOW_LEFT, 0), - max_seqlen_k=seq_len, - ) - out = flash_attn_with_kvcache( - q=q, - k_cache=k_cache, - v_cache=v_cache, - page_table=torch.arange( - k_cache.shape[0], device=dev, dtype=torch.int32 - ).view(1, -1), - cache_seqlens=torch.tensor([seq_len], device=dev, dtype=torch.int32), - cu_seqlens_q=torch.tensor([0, seq_len], device=dev, dtype=torch.int32), - max_seqlen_q=seq_len, - softmax_scale=scale, - causal=block_ids is None, - window_size=window_size, - bidirectional_block_ids=block_ids, - ) - torch.xpu.synchronize() - rows = torch.arange(seq_len, device=dev).view(-1, 1) - cols = torch.arange(seq_len, device=dev).view(1, -1) - return out.float(), q, k, v, rows, cols, scale - - def test_prompt_longer_than_window_keeps_local_attention(self): - """Keys beyond the sliding window must stay masked once the prompt outgrows - the window, even though the batch carries image block ids.""" - out, q, k, v, rows, cols, scale = self._run(seq_len=200) - local = (cols <= rows) & (rows - cols <= self.WINDOW_LEFT) - want = _reference(q, k, v, local, scale) - torch.testing.assert_close(out, want, rtol=2e-2, atol=2e-2) - - def test_prompt_within_window_keeps_bidirectional_image(self): - """At kv length == left window + 1 the window clips nothing, so the image - span must still be attended bidirectionally.""" - out, q, k, v, rows, cols, scale = self._run(seq_len=self.WINDOW_LEFT + 1) - allow = cols <= rows - begin, last = self.SPAN - allow[begin : last + 1, begin : last + 1] = True - want = _reference(q, k, v, allow, scale) - torch.testing.assert_close(out, want, rtol=2e-2, atol=2e-2) - - -@unittest.skipUnless( - hasattr(torch, "xpu") and torch.xpu.is_available(), "requires an Intel XPU" -) -class TestXPUGemma3ImageAfterCachedPrefix(CustomTestCase): - def test_matches_full_bidirectional_attention(self): - """With a cached prefix (radix hit or an earlier chunk), image tokens in the - extend must attend bidirectionally and see the whole prefix. The batch mixes - a page-aligned prefix, an unaligned prefix with an image straddling it, and - no prefix with an image cut by the chunk end.""" - from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend - - torch.manual_seed(0) - dev = "xpu" - scale = D**-0.5 - prefix_lens, extend_lens = [128, 50, 0], [100, 90, 70] - image_spans = [[(140, 190)], [(30, 100)], [(40, 90)]] - seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] - pages_per_seq = [(s + PAGE - 1) // PAGE for s in seq_lens] - num_pages = sum(pages_per_seq) - - # Hand pages out in reverse so each sequence's pages are non-contiguous. - free_pages = list(range(num_pages))[::-1] - req_to_token = torch.zeros(len(seq_lens), 256, dtype=torch.int32, device=dev) - for i, n in enumerate(pages_per_seq): - pages = torch.tensor([free_pages.pop(0) for _ in range(n)], device=dev) - slots = (pages[:, None] * PAGE + torch.arange(PAGE, device=dev)).flatten() - req_to_token[i, : n * PAGE] = slots.to(torch.int32) - k_flat = torch.randn(num_pages * PAGE, HKV, D, device=dev, dtype=torch.bfloat16) - v_flat = torch.randn_like(k_flat) - q = torch.randn(sum(extend_lens), H, D, device=dev, dtype=torch.bfloat16) - - backend = XPUAttentionBackend.__new__(XPUAttentionBackend) - backend.num_splits = 0 - to_dev = lambda x: torch.tensor(x, device=dev, dtype=torch.int32) - block_ids = _build_block_ids( - _image_batch(prefix_lens, extend_lens, image_spans), dev - ) - out = backend._forward_image_blocks_after_prefix( - q=q, - key_cache=k_flat.view(num_pages, PAGE, HKV, D), - value_cache=v_flat.view(num_pages, PAGE, HKV, D), - page_table=req_to_token[:, ::PAGE] // PAGE, - forward_batch=SimpleNamespace( - extend_prefix_lens_cpu=prefix_lens, - extend_prefix_lens=to_dev(prefix_lens), - extend_seq_lens=to_dev(extend_lens), - ), - metadata=SimpleNamespace( - cache_seqlens_int32=to_dev(seq_lens), - cu_seqlens_k=torch.nn.functional.pad( - torch.cumsum(to_dev(seq_lens), dim=0, dtype=torch.int32), (1, 0) - ), - max_seq_len_k=max(seq_lens), - ), - block_ids=block_ids, - scale=scale, - softcap=0.0, - ) - torch.xpu.synchronize() - - q_start = 0 - for i, (prefix_len, seq_len) in enumerate(zip(prefix_lens, seq_lens)): - slots = req_to_token[i, :seq_len].long() - rows = torch.arange(prefix_len, seq_len, device=dev).view(-1, 1) - cols = torch.arange(seq_len, device=dev).view(1, -1) - allow = cols <= rows - for begin, last in image_spans[i]: - allow |= ( - (rows >= begin) & (rows <= last) & (cols >= begin) & (cols <= last) - ) - q_end = q_start + seq_len - prefix_len - want = _reference( - q[q_start:q_end], k_flat[slots], v_flat[slots], allow, scale - ) - torch.testing.assert_close( - out[q_start:q_end].float(), want, rtol=2e-2, atol=2e-2 - ) - q_start = q_end - - -class TestXPUGemma3ImageBlockIds(CustomTestCase): - def test_image_only_in_cached_prefix_needs_no_mask(self): - """A text-only extend after a cached image is exact under causal attention - and must take the plain causal path.""" - self.assertIsNone( - _build_block_ids(_image_batch([256], [50], [[(10, 200)]]), "cpu") - ) - - -if __name__ == "__main__": - unittest.main()