From f03a183719c9e7fb2bd528ed5c7e3b627d9df92d Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 1 Oct 2026 11:53:31 -0400 Subject: [PATCH] Fix Kimi-K3 MLA output gate dispatch on non-CUDA devices (#36406) Co-authored-by: Alex Nails --- .../kernels/ops/attention/mla_output_gate.py | 5 ++--- .../models/test_kimi_k3_mla_output_gate.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/models/test_kimi_k3_mla_output_gate.py diff --git a/python/sglang/kernels/ops/attention/mla_output_gate.py b/python/sglang/kernels/ops/attention/mla_output_gate.py index 721f82b1b4..30634a3c4b 100644 --- a/python/sglang/kernels/ops/attention/mla_output_gate.py +++ b/python/sglang/kernels/ops/attention/mla_output_gate.py @@ -12,13 +12,11 @@ from sglang.kernels.jit.utils import ( load_jit, make_cpp_args, ) -from sglang.srt.utils import is_npu if TYPE_CHECKING: from tvm_ffi.module import Module _THREADS: int = 256 -_is_npu = is_npu() @cache_once @@ -35,7 +33,8 @@ def _jit_mla_output_gate_module() -> Module: def covered(x: torch.Tensor, gate: torch.Tensor) -> bool: return ( - not _is_npu + x.is_cuda + and gate.is_cuda and x.dtype == torch.bfloat16 and gate.dtype == torch.bfloat16 and x.shape == gate.shape diff --git a/test/registered/unit/models/test_kimi_k3_mla_output_gate.py b/test/registered/unit/models/test_kimi_k3_mla_output_gate.py new file mode 100644 index 0000000000..be06617f0d --- /dev/null +++ b/test/registered/unit/models/test_kimi_k3_mla_output_gate.py @@ -0,0 +1,21 @@ +import unittest + +import torch + +from sglang.kernels.ops.attention.mla_output_gate import covered +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +class TestKimiK3MlaOutputGate(CustomTestCase): + def test_cpu_tensors_use_fallback(self): + x = torch.zeros(2, 8, dtype=torch.bfloat16) + gate = torch.zeros_like(x) + + self.assertFalse(covered(x, gate)) + + +if __name__ == "__main__": + unittest.main()