From 56fdfc3b26010ee9889a6498ae963260c4dad711 Mon Sep 17 00:00:00 2001 From: Xia Weiwen Date: Thu, 27 Aug 2026 02:38:02 -0700 Subject: [PATCH] XPU: remove SGLANG_USE_SGL_XPU flag (#34492) --- python/sglang/srt/layers/attention/vision.py | 3 +- .../moe/moe_runner/triton_utils/fused_moe.py | 6 +- python/sglang/srt/layers/moe/utils.py | 4 ++ python/sglang/srt/layers/quantization/fp8.py | 7 +-- .../sglang/srt/layers/quantization/unquant.py | 25 +++++---- python/sglang/srt/server_args.py | 1 + python/sglang/srt/utils/common.py | 4 -- .../llm_models/test_xpu_gemma_4_26b_a4b.py | 1 - .../test_xpu_nemotron_3_nano_30b_a3b.py | 1 - .../xpu/llm_models/test_xpu_qwen3_30b_a3b.py | 2 - .../llm_models/test_xpu_qwen3_5_35b_a3b.py | 2 - test/registered/xpu/test_deepseek_ocr.py | 1 - .../xpu/test_deepseek_ocr_2_olmbench.py | 2 +- .../xpu/test_deepseek_ocr_triton.py | 3 +- .../xpu/test_encoder_attention_backend.py | 2 - test/registered/xpu/test_gemma_4_e2b.py | 2 - test/registered/xpu/test_intel_xpu_backend.py | 2 + test/registered/xpu/test_moe_ld_padding.py | 19 ++++--- .../xpu/test_vision_backend_selection.py | 55 +++++++++++++++++++ 19 files changed, 96 insertions(+), 46 deletions(-) create mode 100644 test/registered/xpu/test_vision_backend_selection.py diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index 5e3e8c6f3..20e87619d 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -30,7 +30,6 @@ from sglang.srt.utils import ( is_npu, is_xpu, print_info_once, - use_intel_xpu_backend, ) from sglang.srt.utils.multi_stream_utils import ( maybe_execute_in_parallel, @@ -1266,7 +1265,7 @@ class VisionAttention(nn.Module): elif _is_cpu and _is_cpu_amx_available: backend = "amx_attn" elif _is_xpu: - backend = "triton_attn" if not use_intel_xpu_backend() else "xpu_attn" + backend = "xpu_attn" else: backend = "sdpa" if backend == "fa3" and is_blackwell_supported(): diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index 546f79b58..758c2eacf 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -28,7 +28,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig -from sglang.srt.layers.moe.utils import get_moe_padding_size +from sglang.srt.layers.moe.utils import get_moe_padding_size, get_moe_runner_backend from sglang.srt.runtime_context import get_exec from sglang.srt.utils import ( cpu_has_amx_support, @@ -38,7 +38,6 @@ from sglang.srt.utils import ( is_hip, is_musa, is_xpu, - use_intel_xpu_backend, ) from sglang.srt.utils.custom_op import register_custom_op @@ -54,7 +53,6 @@ _is_cpu_amx_available = cpu_has_amx_support() _is_cpu = is_cpu() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip _is_xpu = is_xpu() -_use_sgl_xpu = use_intel_xpu_backend() _is_musa = is_musa() @@ -1133,7 +1131,7 @@ def fused_moe( Returns: - torch.Tensor: The output tensor after applying the MoE layer. """ - if _use_sgl_xpu: + if _is_xpu and not get_moe_runner_backend().is_triton(): topk_weight, topk_ids, _ = topk_output from sgl_kernel import fused_experts as sgl_fused_experts diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 42a653cf7..cb1e2ad6f 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -119,6 +119,7 @@ class MoeRunnerBackend(Enum): EXPERIMENTAL_SGL_MARLIN = "experimental_sgl_marlin" AITER = "aiter" HPC_OPS = "hpc_ops" + INTEL_XPU = "intel_xpu" def is_auto(self): return self == MoeRunnerBackend.AUTO @@ -182,6 +183,9 @@ class MoeRunnerBackend(Enum): def is_aiter(self): return self == MoeRunnerBackend.AITER + def is_intel_xpu(self): + return self == MoeRunnerBackend.INTEL_XPU + class DeepEPv2Fp8ScaleFormat(NamedTuple): """DeepGEMM FP8 activation-scale layout expected from DeepEP v2.""" diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index be46c4960..b449988af 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -95,12 +95,12 @@ from sglang.srt.utils import ( is_sm90_supported, is_sm100_supported, is_sm120_supported, + is_xpu, log_info_on_rank0, mxfp8_block_convert_required, print_warning_once, set_weight_attrs, use_intel_amx_backend, - use_intel_xpu_backend, ) if TYPE_CHECKING: @@ -2435,10 +2435,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): if quant_info is not None: return self.runner.run(dispatch_output, quant_info) - if use_intel_xpu_backend() and not ( - getattr(self, "runner", None) is not None - and self.runner.runner_backend.is_triton() - ): + if is_xpu() and not get_moe_runner_backend().is_triton(): # sgl-kernel-xpu path from sgl_kernel import fused_experts diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 8e7b934d9..eaf44ee03 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -42,9 +42,9 @@ from sglang.srt.utils import ( is_cuda, is_hip, is_npu, + is_xpu, set_weight_attrs, use_intel_amx_backend, - use_intel_xpu_backend, ) from sglang.srt.utils.custom_op import register_custom_op @@ -343,18 +343,20 @@ class UnquantizedLinearMethod(LinearMethodBase): def _use_xpu_moe_ld_padding(use_triton_kernels: bool) -> bool: """Whether MoE expert weights should get a padded row stride for XPU. - use_intel_xpu_backend() only tells us an XPU exists on this machine, not - that the weights being created land on it -- the env var can be set while - serving on CPU/CUDA. create_weights takes no device argument and allocates - under the model loader's ambient device context, so check that context too: - padding a non-XPU weight would make it non-contiguous for no benefit, and - other backends' MoE kernels expect contiguous expert tensors. + is_xpu() only tells us an XPU exists on this machine, not that the weights + being created land on it -- this can be true while serving on CPU/CUDA. + create_weights takes no device argument and allocates under the model + loader's ambient device context, so check that context too: padding a + non-XPU weight would make it non-contiguous for no benefit, and other + backends' MoE kernels expect contiguous expert tensors. The Triton path stores B transposed and does not read a row stride, so it - is excluded even on XPU. + is excluded even on XPU (either via --moe-runner-backend triton or the + triton_kernels build). """ return ( - use_intel_xpu_backend() + is_xpu() + and not get_moe_runner_backend().is_triton() and torch.get_default_device().type == "xpu" and not use_triton_kernels ) @@ -948,7 +950,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): ], f"activation = {moe_runner_config.activation} is not supported." backend = self.runner.runner_backend - if use_intel_xpu_backend(): + if not get_moe_runner_backend().is_triton(): # sgl-kernel-xpu path from sgl_kernel import fused_experts @@ -974,7 +976,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): assert ( moe_runner_config.activation == "silu" ), f"activation = {moe_runner_config.activation} is not supported \ - for Triton PATH, please set ENV SGLANG_USE_SGL_XPU=1." + for Triton PATH, please drop --moe-runner-backend triton to use \ + the sgl-kernel-xpu path, which supports more activations." quant_info = self.get_triton_quant_info(layer) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index be0455e03..e59d294e9 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -305,6 +305,7 @@ MOE_RUNNER_BACKEND_CHOICES = [ "experimental_sgl_marlin", "hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), FP8 MoE on Hopper (SM90) only "megamoe", + "intel_xpu", ] add_moe_runner_backend_choices = MOE_RUNNER_BACKEND_CHOICES.extend diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 1f6cdfef7..9c04be7f1 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -354,10 +354,6 @@ def xpu_has_xmx_support(): return False -def use_intel_xpu_backend(): - return get_bool_env_var("SGLANG_USE_SGL_XPU") and is_xpu() - - @lru_cache(maxsize=1) def is_flashinfer_available(): """ diff --git a/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py b/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py index 156888af8..f7f349cbe 100644 --- a/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py +++ b/test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py @@ -23,7 +23,6 @@ class TestGemma4_26BA4BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): tp_size = 4 accuracy = 0.90 timeout_for_server_launch = 3600 - env = {"SGLANG_USE_SGL_XPU": "1"} # Gemma-4 hybrid-attention kernels crash under chunked prefill on XPU. other_args = SimpleEvalGSM8KXPUMixin.other_args + [ diff --git a/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py b/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py index 174099c15..7aefd1350 100644 --- a/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py +++ b/test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py @@ -27,7 +27,6 @@ class TestNemotron3Nano30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): max_tokens = 8192 # Client-side eval concurrency (mixin default is 1). num_threads = 4 - env = {"SGLANG_USE_SGL_XPU": "1"} # Hybrid-mamba layout needs --model-impl sglang, a fixed page size, and # the nemotron_3 reasoning / qwen3_coder tool-call parsers. diff --git a/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py b/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py index 1548b950b..e5992953e 100644 --- a/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py +++ b/test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py @@ -23,8 +23,6 @@ class TestQwen3_30BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): tp_size = 4 accuracy = 0.90 timeout_for_server_launch = 3600 - # SGL XPU MoE kernels gate on this env var. - env = {"SGLANG_USE_SGL_XPU": "1"} num_examples = 50 num_threads = 4 max_tokens = 8192 diff --git a/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py b/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py index 0d5befc28..23caeb904 100644 --- a/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py +++ b/test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py @@ -23,8 +23,6 @@ class TestQwen3_5_35BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): tp_size = 4 accuracy = 0.90 timeout_for_server_launch = 3600 - # SGL XPU MoE kernels gate on this env var. - env = {"SGLANG_USE_SGL_XPU": "1"} num_examples = 50 num_threads = 4 max_tokens = 8192 diff --git a/test/registered/xpu/test_deepseek_ocr.py b/test/registered/xpu/test_deepseek_ocr.py index 5e8792829..31a5df7a6 100644 --- a/test/registered/xpu/test_deepseek_ocr.py +++ b/test/registered/xpu/test_deepseek_ocr.py @@ -43,7 +43,6 @@ class TestDeepSeekOCR(CustomTestCase): "--attention-backend", "intel_xpu", ] - os.environ["SGLANG_USE_SGL_XPU"] = "1" cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/xpu/test_deepseek_ocr_2_olmbench.py b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py index 7e3eb5dc5..48ddc1b9a 100644 --- a/test/registered/xpu/test_deepseek_ocr_2_olmbench.py +++ b/test/registered/xpu/test_deepseek_ocr_2_olmbench.py @@ -67,7 +67,7 @@ class TestDeepSeekOCR2OlmBenchXPU(CustomTestCase): "8192", "--disable-cuda-graph", ] - env = {"SGLANG_USE_SGL_XPU": "1"} + env = {} @classmethod def setUpClass(cls): diff --git a/test/registered/xpu/test_deepseek_ocr_triton.py b/test/registered/xpu/test_deepseek_ocr_triton.py index b17c29710..7bde8bd33 100644 --- a/test/registered/xpu/test_deepseek_ocr_triton.py +++ b/test/registered/xpu/test_deepseek_ocr_triton.py @@ -41,8 +41,9 @@ class TestDeepSeekOCRTriton(TestDeepSeekOCR): "xpu", "--attention-backend", "intel_xpu", + "--moe-runner-backend", + "triton", ] - os.environ["SGLANG_USE_SGL_XPU"] = "0" cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/xpu/test_encoder_attention_backend.py b/test/registered/xpu/test_encoder_attention_backend.py index 1f64c15df..cfeee33fc 100644 --- a/test/registered/xpu/test_encoder_attention_backend.py +++ b/test/registered/xpu/test_encoder_attention_backend.py @@ -40,7 +40,6 @@ class TestEncoderAttention(CustomTestCase): "--mm-attention-backend", "xpu_attn", ] - os.environ["SGLANG_USE_SGL_XPU"] = "1" cls.process = popen_launch_server( cls.model, cls.base_url, @@ -128,7 +127,6 @@ class TestEncoderAttention_Triton(TestEncoderAttention): "--mm-attention-backend", "triton_attn", ] - os.environ["SGLANG_USE_SGL_XPU"] = "0" cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/xpu/test_gemma_4_e2b.py b/test/registered/xpu/test_gemma_4_e2b.py index a340a58bc..08ef013d4 100644 --- a/test/registered/xpu/test_gemma_4_e2b.py +++ b/test/registered/xpu/test_gemma_4_e2b.py @@ -19,7 +19,6 @@ Server is started with ``sglang serve`` (``--model-impl sglang``). from __future__ import annotations -import os import unittest import openai @@ -83,7 +82,6 @@ class TestGemma4E2BXPU(CustomTestCase): cls.model = MODEL cls.base_url = DEFAULT_URL_FOR_TEST cls.api_key = "sk-123456" - os.environ["SGLANG_USE_SGL_XPU"] = "1" _empty_xpu_cache() cls.process = popen_launch_server( diff --git a/test/registered/xpu/test_intel_xpu_backend.py b/test/registered/xpu/test_intel_xpu_backend.py index b38c57984..0ef8e088f 100644 --- a/test/registered/xpu/test_intel_xpu_backend.py +++ b/test/registered/xpu/test_intel_xpu_backend.py @@ -75,6 +75,8 @@ class TestIntelXPUBackend(CustomTestCase): '{"num_hidden_layers": 4}', "--decode-attention-backend", "intel_xpu", + "--moe-runner-backend", + "triton", # FP8 is not yet supported in sgl-kernel ], min_throughput=32, ) diff --git a/test/registered/xpu/test_moe_ld_padding.py b/test/registered/xpu/test_moe_ld_padding.py index ca9cf0d63..92cefa48a 100644 --- a/test/registered/xpu/test_moe_ld_padding.py +++ b/test/registered/xpu/test_moe_ld_padding.py @@ -9,6 +9,7 @@ import torch from sglang.srt.layers.moe.utils import ( XPU_MOE_LD_PADDING_BYTES, + MoeRunnerBackend, xpu_moe_ld_padding_elems, ) from sglang.srt.layers.quantization.unquant import _empty_xpu_moe_expert_weight @@ -60,7 +61,7 @@ class TestXpuMoeLdPadding(CustomTestCase): self.assertTrue(unpadded.is_contiguous()) def test_only_pads_weights_that_land_on_xpu(self): - # SGLANG_USE_SGL_XPU only says an XPU exists on the machine; the weights + # is_xpu() only says an XPU exists on the machine; the weights # can still be built for CPU/CUDA. create_weights takes no device # argument, so the gate reads the ambient device context. Padding a # non-XPU weight would make it non-contiguous for no benefit. @@ -88,10 +89,12 @@ class TestXpuMoeLdPadding(CustomTestCase): return layer.w13_weight, layer.w2_weight with unittest.mock.patch( - "sglang.srt.layers.quantization.unquant.use_intel_xpu_backend", - return_value=True, + "sglang.srt.layers.quantization.unquant.is_xpu", return_value=True + ), unittest.mock.patch( + "sglang.srt.layers.quantization.unquant.get_moe_runner_backend", + return_value=MoeRunnerBackend.AUTO, ): - # Env var on but building for CPU -> must stay contiguous. + # Backend on but building for CPU -> must stay contiguous. w13_cpu, w2_cpu = build("cpu") self.assertTrue(w13_cpu.is_contiguous()) self.assertTrue(w2_cpu.is_contiguous()) @@ -103,10 +106,12 @@ class TestXpuMoeLdPadding(CustomTestCase): w13_triton, _ = build("xpu", use_triton_kernels=True) self.assertTrue(w13_triton.is_contiguous()) - # Backend off entirely -> never padded, even on XPU. + # Backend forced to Triton -> never padded, even on XPU. with unittest.mock.patch( - "sglang.srt.layers.quantization.unquant.use_intel_xpu_backend", - return_value=False, + "sglang.srt.layers.quantization.unquant.is_xpu", return_value=True + ), unittest.mock.patch( + "sglang.srt.layers.quantization.unquant.get_moe_runner_backend", + return_value=MoeRunnerBackend.TRITON, ): device = "xpu" if torch.xpu.is_available() else "cpu" w13, w2 = build(device) diff --git a/test/registered/xpu/test_vision_backend_selection.py b/test/registered/xpu/test_vision_backend_selection.py new file mode 100644 index 000000000..6245c78e2 --- /dev/null +++ b/test/registered/xpu/test_vision_backend_selection.py @@ -0,0 +1,55 @@ +import sys +from types import SimpleNamespace + +import pytest + +from sglang.srt.layers.attention import vision +from sglang.test.ci.ci_register import register_xpu_ci + +register_xpu_ci(est_time=300, suite="stage-a-test-1-gpu-xpu") + + +@pytest.mark.parametrize( + ("server_backend", "passed_backend", "expected"), + [ + ( + None, + None, + "xpu_attn", + ), # server backend is not set, expected to use xpu_attn as default + ( + "xpu_attn", + None, + "xpu_attn", + ), # server backend is set to xpu_attn, expected to use xpu_attn + ( + "xpu_attn", + "triton_attn", + "xpu_attn", + ), # server backend is set to xpu_attn, passed backend is triton_attn + ( + "triton_attn", + None, + "triton_attn", + ), # server backend is set to triton_attn, expected to use triton_attn + ], +) +def test_xpu_backend_selection_priority( + monkeypatch, + server_backend, # specified by the server argument + passed_backend, # specified by the layer argument + expected, +): + monkeypatch.setattr( + vision, + "get_mm", + lambda: SimpleNamespace(mm_attention_backend=server_backend), + ) + + backend = vision.VisionAttention._determine_attention_backend(None, passed_backend) + + assert backend == expected + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__]))