diff --git a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py index 1824fcd964..7896812278 100644 --- a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py +++ b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py @@ -250,9 +250,9 @@ def fused_mamba_state_scatter_with_mask( raise ValueError( f"dst and src must be on the same device. {dst.device=} {src.device=}" ) - if not dst.is_cuda or not src.is_cuda: + if not (dst.is_cuda or dst.is_xpu) or dst.device != src.device: raise ValueError( - "fused_mamba_state_scatter_with_mask only supports CUDA tensors." + "fused_mamba_state_scatter_with_mask only supports CUDA/XPU tensors." ) if dst.ndim < 2 or src.ndim < 3: raise ValueError(f"Unexpected tensor ranks: {dst.ndim=} {src.ndim=}") @@ -410,10 +410,10 @@ def fused_conv_window_scatter_with_mask( if total_requests == 0: return - if not (dst.is_cuda and src.is_cuda and dst.device == src.device): + if not (dst.is_cuda or dst.is_xpu) or dst.device != src.device: raise ValueError( - "fused_conv_window_scatter_with_mask requires dst and src to be CUDA " - f"tensors on the same device ({dst.device=}, {src.device=})." + "fused_conv_window_scatter_with_mask requires dst and src to be " + f"CUDA/XPU tensors on the same device ({dst.device=}, {src.device=})." ) if dst.ndim != 4 or src.ndim != 5: raise ValueError(f"Unexpected ranks: {dst.ndim=} (want 4) {src.ndim=} (want 5)") diff --git a/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py index 4b2499bdd5..8568b874ff 100644 --- a/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py @@ -95,6 +95,11 @@ def fused_sigmoid_gating_delta_rule_update( o=o, h0_source=initial_state_source, h0_indices=initial_state_indices, + # Envelope-strided state pools (page-major / unified memory) have a + # per-slot pitch != HV*K*V; contiguous pools pass exactly HV*K*V. + stride_h0_source=( + initial_state_source.stride(0) if initial_state_source is not None else 0 + ), cu_seqlens=cu_seqlens, intermediate_states_buffer=intermediate_states_buffer, intermediate_state_indices=intermediate_state_indices, diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index a155858351..618981b245 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -519,13 +519,16 @@ class QSAIndexer(MultiPlatformOp): token_topk=self.token_topk, ) - def forward_cuda( + def _forward_impl( self, hidden_states: torch.Tensor, positions: torch.Tensor, forward_batch, indexer_metadata, ) -> torch.Tensor: + """Portable orchestration shared by different platforms. + Fast paths are gated per platforms inside kernel calls. + """ forward_mode = forward_batch.forward_mode is_target_verify = getattr(forward_mode, "is_target_verify", lambda: False)() is_draft_extend = getattr(forward_mode, "is_draft_extend_v2", lambda: False)() @@ -628,6 +631,28 @@ class QSAIndexer(MultiPlatformOp): row_sequence_lengths, ) + def forward_cuda( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + forward_batch, + indexer_metadata, + ) -> torch.Tensor: + return self._forward_impl( + hidden_states, positions, forward_batch, indexer_metadata + ) + + def forward_xpu( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + forward_batch, + indexer_metadata, + ) -> torch.Tensor: + return self._forward_impl( + hidden_states, positions, forward_batch, indexer_metadata + ) + __all__ = [ "QSAIndexer", diff --git a/python/sglang/srt/layers/hyperconnection.py b/python/sglang/srt/layers/hyperconnection.py index 61516924da..c9439db590 100644 --- a/python/sglang/srt/layers/hyperconnection.py +++ b/python/sglang/srt/layers/hyperconnection.py @@ -139,14 +139,14 @@ class GatedResidual(HyperConnectionBase): self.hidden_size * self.hc_count, self.config.hc_lowrank, bias=False, - device=torch.cuda.current_device(), + device=torch.get_device_module().current_device(), dtype=config.params_dtype, ) self.input_mix_weight_up = nn.Linear( self.config.hc_lowrank, self.hc_count * self.hidden_size, bias=False, - device=torch.cuda.current_device(), + device=torch.get_device_module().current_device(), dtype=config.params_dtype, ) lowrank = self.config.hc_lowrank @@ -166,7 +166,7 @@ class GatedResidual(HyperConnectionBase): self.hidden_size * self.hc_count, self.hc_count, bias=False, - device=torch.cuda.current_device(), + device=torch.get_device_module().current_device(), dtype=config.params_dtype, ) # hc_combine rejects other shapes; device and dtype are checked per call. diff --git a/test/registered/kernels/ops/qsa/test_qsa.py b/test/registered/kernels/ops/qsa/test_qsa.py index 8bb75e6255..6038bae1ab 100644 --- a/test/registered/kernels/ops/qsa/test_qsa.py +++ b/test/registered/kernels/ops/qsa/test_qsa.py @@ -1,5 +1,5 @@ import sys -from types import ModuleType, SimpleNamespace +from types import MethodType, ModuleType, SimpleNamespace import pytest import torch @@ -566,6 +566,7 @@ def test_qsa_indexer_ignores_dp_attention_token_padding(): q.squeeze(1) ), ) + indexer._forward_impl = MethodType(QSAIndexer._forward_impl, indexer) forward_batch = SimpleNamespace( forward_mode=ForwardMode.EXTEND, positions=torch.cat( @@ -715,6 +716,7 @@ def test_qsa_indexer_rejects_shorter_source_than_request_mapping(): get_token_to_batch_idx=lambda: torch.zeros(16, dtype=torch.int32) ) indexer = SimpleNamespace() + indexer._forward_impl = MethodType(QSAIndexer._forward_impl, indexer) batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND, positions=torch.arange(15)) try: QSAIndexer.forward_cuda( @@ -960,6 +962,9 @@ class _DispatchIndexer: index_n_heads = 4 compress_ratio = 4 _pending_ring_slots = QSAIndexer._pending_ring_slots + # forward_cuda (the code under test) delegates to _forward_impl; bind the + # real implementation so this mock indexer can be dispatched through it. + _forward_impl = QSAIndexer._forward_impl def __init__(self): self.selected = None diff --git a/test/registered/xpu/test_qsa_indexer_xpu.py b/test/registered/xpu/test_qsa_indexer_xpu.py new file mode 100644 index 0000000000..310d4ed797 --- /dev/null +++ b/test/registered/xpu/test_qsa_indexer_xpu.py @@ -0,0 +1,238 @@ +"""XPU unit tests for the Qwen3.8-Flash-Next QSA (sparse-attention) indexer. + +Mirrors test/registered/kernel/qsa/test_qsa.py's dispatch coverage, plus a +real-hardware regression test for the compress-group OOB gather (#38346). + +A full E2E integration test would need real model weights or the reduced +dummy checkpoint used for local validation (neither is published). +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer +from sglang.test.ci.ci_register import register_xpu_ci + +register_xpu_ci(est_time=20, suite="stage-b-test-1-gpu-xpu") + + +class _ForwardMode: + def __init__(self, decode): + self.decode = decode + + def is_decode(self): + return self.decode + + +class _DispatchIndexer: + """Mirrors test_qsa.py's _DispatchIndexer; used to prove forward_xpu + dispatches prefill/decode identically to QSAIndexer's behavior.""" + + layer_id = 3 + index_n_heads = 4 + compress_ratio = 4 + _pending_ring_slots = QSAIndexer._pending_ring_slots + _forward_impl = QSAIndexer._forward_impl + + def __init__(self): + self.selected = None + self.logical_positions = None + + @staticmethod + def project_qk(hidden_states, positions, **kwargs): + rows = hidden_states.shape[0] + return torch.zeros(rows, 4, 128), torch.zeros(rows, 1, 128), False + + def select_prefill_tokens(self, *args): + self.selected = "prefill" + return torch.tensor([1]) + + def select_decode_tokens(self, *args): + self.selected = "decode" + return torch.tensor([2]) + + def update_key_state_and_compress( + self, token_k, logical_positions, rope_positions, metadata, **kwargs + ): + self.logical_positions = logical_positions.clone() + + +class _DispatchMetadata: + token_to_kv_pool = None + out_cache_loc = None + compress_member_rows = None + decode_logical_positions = None + pending_ring_slots = None + # Consumed by the real _pending_ring_slots helper the dispatch indexer + # borrows: one token row owned by request slot 1. + token_to_batch_idx = torch.zeros(2, dtype=torch.int32) + req_pool_indices = torch.ones(2, dtype=torch.int32) + sequence_lengths = torch.ones(2, dtype=torch.int32) + + @staticmethod + def get_decode_mqa_inputs(layer_id): + return ( + torch.zeros(2, 1, 1, 128), + torch.zeros(1, 1, dtype=torch.int32), + torch.ones(1, dtype=torch.int32), + 1, + ) + + @staticmethod + def get_prefill_mqa_inputs(layer_id, positions): + return ( + torch.zeros(1, 1, 128), + torch.zeros(1, dtype=torch.int32), + torch.ones(1, dtype=torch.int32), + torch.ones(1, dtype=torch.int32), + ) + + @staticmethod + def get_token_to_batch_idx(): + return torch.zeros(1, dtype=torch.int32) + + @staticmethod + def get_seqlens_int32(): + return torch.ones(1, dtype=torch.int32) + + @staticmethod + def get_seqlens_expanded(): + return torch.tensor([7], dtype=torch.int32) + + +class TestQSAIndexerForwardXpuDispatch(unittest.TestCase): + """forward_xpu must route prefill/decode identically to QSAIndexer""" + + def test_forward_xpu_dispatches_prefill_and_decode_mqa(self): + indexer = _DispatchIndexer() + metadata = _DispatchMetadata() + inputs = torch.zeros(1, 16) + positions = torch.zeros(1, dtype=torch.int32) + + prefill_result = QSAIndexer.forward_xpu( + indexer, + inputs, + positions, + SimpleNamespace(forward_mode=_ForwardMode(False)), + metadata, + ) + self.assertEqual(indexer.selected, "prefill") + self.assertEqual(indexer.logical_positions.tolist(), [0]) + self.assertEqual(prefill_result.item(), 1) + + positions.fill_(99) # an arbitrary position for decode mode + decode_result = QSAIndexer.forward_xpu( + indexer, + inputs, + positions, + SimpleNamespace(forward_mode=_ForwardMode(True)), + metadata, + ) + self.assertEqual(indexer.selected, "decode") + # It should be _DispatchMetadata.get_seqlens_expanded() - 1 + self.assertEqual(indexer.logical_positions.tolist(), [6]) + # It should be _DispatchIndexer.select_decode_tokens(...) + self.assertEqual(decode_result.item(), 2) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestQSACompressGroupOOBClampOnRealXpu(unittest.TestCase): + """Regression test for the compress-group OOB gather (#38346): calls + ``QSAIndexer.update_key_state_and_compress`` with a short extend + chunk that forces ``group_locs`` past the end of ``token_k``. + Unclamped, XPU's bounds-checked SYCL gather aborts the process.""" + + def _build_indexer(self): + from sglang.srt.layers.rotary_embedding import get_rope + + config = SimpleNamespace( + hidden_size=32, + indexer_n_heads=2, + indexer_kv_heads=1, + indexer_head_dim=8, + indexer_budget=2048, + indexer_compress_ratio=4, + rms_norm_eps=1e-6, + ) + rotary_emb = get_rope( + head_size=8, + rotary_dim=8, + max_position=64, + base=10000, + is_neox_style=True, + dtype=torch.float32, + ) + indexer = QSAIndexer(config, layer_id=0, rotary_emb=rotary_emb) + return indexer.to("xpu") + + def test_update_key_state_and_compress_clamps_oob_group_locs(self): + indexer = self._build_indexer() + num_rows = 6 # token_k has only 6 valid rows in this extend chunk. + token_k = torch.randn(num_rows, 1, 8, dtype=torch.float32, device="xpu") + + # A worst-case capacity bound can reserve more groups than this short + # chunk actually has complete members for; the last group's starting + # row, uncompensated, walks the real gather past num_rows - 1. + compressed_k = torch.zeros(3, 1, 8, dtype=torch.float32, device="xpu") + metadata = SimpleNamespace( + token_to_kv_pool=SimpleNamespace( + set_qsa_compressed_k_buffer=lambda layer_id, loc, val: ( + compressed_k.__setitem__(loc.long(), val) + ), + ), + compress_member_rows=torch.tensor( + [0, 4, 5], dtype=torch.long, device="xpu" + ), + is_cuda_graph=False, + write_locs=torch.arange(3, dtype=torch.long, device="xpu"), + compress_group_positions=torch.zeros(3, dtype=torch.long, device="xpu"), + extend_rope_matrix=None, + ) + + # Must not raise/abort (this is the real production code path). + indexer.update_key_state_and_compress( + token_k, + logical_positions=torch.arange(num_rows, device="xpu"), + rope_positions=torch.arange(num_rows, device="xpu"), + metadata=metadata, + state_slots=None, + state_stored=True, # skip the pending-ring write; irrelevant here. + ) + + # The clamped tail group repeats row 5 (the last valid row) instead + # of reading out of bounds. Build an unclamped equivalent -- a chunk + # padded with copies of row 5 so the same group [5, 5, 5, 5] is + # reached without any clamping -- and check both compress to the + # same result. + padded_token_k = torch.cat([token_k, token_k[-1:].expand(3, -1, -1)], dim=0) + reference_k = torch.zeros(1, 1, 8, dtype=torch.float32, device="xpu") + reference_metadata = SimpleNamespace( + token_to_kv_pool=SimpleNamespace( + set_qsa_compressed_k_buffer=lambda layer_id, loc, val: ( + reference_k.__setitem__(loc.long(), val) + ), + ), + compress_member_rows=torch.tensor([5], dtype=torch.long, device="xpu"), + is_cuda_graph=False, + write_locs=torch.zeros(1, dtype=torch.long, device="xpu"), + compress_group_positions=torch.zeros(1, dtype=torch.long, device="xpu"), + extend_rope_matrix=None, + ) + indexer.update_key_state_and_compress( + padded_token_k, + logical_positions=torch.arange(padded_token_k.shape[0], device="xpu"), + rope_positions=torch.arange(padded_token_k.shape[0], device="xpu"), + metadata=reference_metadata, + state_slots=None, + state_stored=True, + ) + torch.testing.assert_close(compressed_k[2], reference_k[0]) + + +if __name__ == "__main__": + unittest.main()