From e3205080ce400eb8ef3d5261fe1ef8c122548ac6 Mon Sep 17 00:00:00 2001 From: wxzhoucs <66296518+Misaka9468@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:32:40 +0800 Subject: [PATCH] [Fix] qwen3.8: Fix batch-invariant kernels for Qwen3.8 Flash-Next (#42087) Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../kernels/ops/elementwise/elementwise.py | 3 +- .../batch_invariant_ops.py | 7 ++ .../sglang/srt/layers/attention/qsa/kernel.py | 16 +++ .../srt/layers/attention/qsa/qsa_indexer.py | 7 +- .../srt/layers/attention/qsa/sparse_attn.py | 4 + .../ops/attention/qsa/test_qsa_determinism.py | 112 ++++++++++++++++++ .../test_qwen_batch_invariance.py | 59 +++++++++ 7 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 test/registered/kernels/ops/attention/qsa/test_qsa_determinism.py create mode 100644 test/registered/unit/batch_invariant_ops/test_qwen_batch_invariance.py diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index 0ed8914b67..325c250d86 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -3,6 +3,7 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.utils import is_hip _is_hip = is_hip() @@ -231,7 +232,7 @@ def _launch_fused_gate_sigmoid_mul( ), } - if num_tokens >= 1024: + if num_tokens >= 1024 or is_batch_invariant_mode_enabled(): config["num_warps"] = min(config["num_warps"], 8) use_pdl = is_arch_support_pdl() diff --git a/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py b/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py index b6a7a5849a..f59c824d22 100644 --- a/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py +++ b/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py @@ -602,6 +602,12 @@ def mm_batch_invariant(a, b): return matmul_persistent(a, b) +def mm_out_batch_invariant( + a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor +) -> torch.Tensor: + return out.copy_(matmul_persistent(a, b)) + + def addmm_batch_invariant(bias, a, b): return matmul_persistent(a, b, bias=bias) @@ -1014,6 +1020,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True): if not _is_npu: # Register for detected device _batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, dispatch_key) + _batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, dispatch_key) _batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, dispatch_key) _batch_invariant_LIB.impl( "aten::_log_softmax", _log_softmax_batch_invariant, dispatch_key diff --git a/python/sglang/srt/layers/attention/qsa/kernel.py b/python/sglang/srt/layers/attention/qsa/kernel.py index 8a24ba4395..07d2316198 100644 --- a/python/sglang/srt/layers/attention/qsa/kernel.py +++ b/python/sglang/srt/layers/attention/qsa/kernel.py @@ -8,6 +8,9 @@ import torch import triton import triton.language as tl +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled +from sglang.srt.utils import is_hip + def average_pool_qsa_keys(key_groups: torch.Tensor) -> torch.Tensor: """FP32-average complete key groups shaped ``[groups, ratio, kv_heads, dim]``.""" @@ -31,6 +34,19 @@ def qsa_fast_topk( lengths = (row_ends - row_starts).to(device=logits.device, dtype=torch.int32) starts = row_starts.to(device=logits.device, dtype=torch.int32) if logits.is_cuda: + if is_batch_invariant_mode_enabled() and not is_hip(): + from flashinfer import TopKTieBreak, top_k_ragged_transform + + return top_k_ragged_transform( + logits, + torch.zeros_like(starts), + lengths, + topk, + deterministic=True, + tie_break=TopKTieBreak.SMALL, + dsa_graph_safe=True, + row_starts=starts, + ) if topk == 512: # Prefer the JIT kernel: it ships with the sglang python package, # so top-k 512 works regardless of the installed sgl_kernel version. diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index 883671944c..0731dfd071 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -6,6 +6,7 @@ from typing import Tuple import torch +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.layers.attention.qsa.kernel import ( average_pool_qsa_keys, expand_qsa_block_indices, @@ -501,7 +502,11 @@ class QSAIndexer(MultiPlatformOp): ) -> torch.Tensor: # fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading # their scores, and never reads past a row's length. - use_fast_topk = q.is_cuda and self.block_topk == 512 + use_fast_topk = ( + q.is_cuda + and self.block_topk == 512 + and not is_batch_invariant_mode_enabled() + ) logits = qsa_mqa_decode( q, compressed_cache, diff --git a/python/sglang/srt/layers/attention/qsa/sparse_attn.py b/python/sglang/srt/layers/attention/qsa/sparse_attn.py index 2aca96d711..8e042e6fb9 100644 --- a/python/sglang/srt/layers/attention/qsa/sparse_attn.py +++ b/python/sglang/srt/layers/attention/qsa/sparse_attn.py @@ -6,6 +6,8 @@ import torch import triton import triton.language as tl +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled + _H20_CONFIGS = [ (32, (32, 8, 2)), (64, (64, 8, 2)), @@ -22,6 +24,8 @@ _L20_CONFIGS = [ def _get_best_config(total_q: int): + if is_batch_invariant_mode_enabled(): + return (32, 4, 2) table = _H20_CONFIGS if "H20" in torch.cuda.get_device_name(0) else _L20_CONFIGS return next(cfg for limit, cfg in table if total_q <= limit) diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_determinism.py b/test/registered/kernels/ops/attention/qsa/test_qsa_determinism.py new file mode 100644 index 0000000000..05138ae50e --- /dev/null +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_determinism.py @@ -0,0 +1,112 @@ +import sys +from collections.abc import Iterator +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.batch_invariant_ops import ( + disable_batch_invariant_mode, + enable_batch_invariant_mode, +) +from sglang.srt.layers.attention.qsa import qsa_indexer as qsa_indexer_module +from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk +from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer +from sglang.srt.layers.attention.qsa.sparse_attn import ( + sparse_gqa_fwd_interface_triton, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="nightly", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + + +@pytest.fixture(autouse=True) +def batch_invariant_mode() -> Iterator[None]: + enable_batch_invariant_mode() + try: + yield + finally: + disable_batch_invariant_mode() + + +@pytest.mark.parametrize("topk", [512, 2048]) +def test_qsa_topk_ragged_ties_and_graph(topk: int) -> None: + torch.manual_seed(42) + lengths = [0, 1, 31, 511, 512, 513, 2047, 2048, 2049] + logits = torch.randint(0, 32, (len(lengths), 4096), device="cuda").float() + starts = torch.arange(len(lengths), dtype=torch.int32, device="cuda") * 37 + ends = starts + torch.tensor(lengths, dtype=torch.int32, device="cuda") + expected = torch.full((len(lengths), topk), -1, dtype=torch.int32, device="cuda") + for row, length in enumerate(lengths): + start = row * 37 + order = torch.argsort( + logits[row, start : start + length], descending=True, stable=True + ) + count = min(length, topk) + expected[row, :count] = order[:count].sort().values.to(torch.int32) + actual = qsa_fast_topk(logits, starts, ends, topk) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = qsa_fast_topk(logits, starts, ends, topk) + graph.replay() + torch.testing.assert_close(captured, expected, rtol=0, atol=0) + + +def test_qsa_decode_topk_repeatability(monkeypatch: pytest.MonkeyPatch) -> None: + torch.manual_seed(42) + logits = torch.randn(16, 2048, device="cuda").round() + + def mock_mqa_decode(*args: object, min_scored_len: int) -> torch.Tensor: + assert min_scored_len == 0 + return logits + + monkeypatch.setattr(qsa_indexer_module, "qsa_mqa_decode", mock_mqa_decode) + indexer = SimpleNamespace(block_topk=512) + lengths = torch.full((16,), 2048, dtype=torch.int32, device="cuda") + results = [ + QSAIndexer.select_decode_tokens( + indexer, + q=logits, + compressed_cache=logits, + compressed_page_table=lengths, + compressed_lengths=lengths, + max_model_len=2048, + query_positions=lengths, + sequence_lengths=lengths, + defer_expansion=True, + ) + for _ in range(4) + ] + for actual in results[1:]: + torch.testing.assert_close(actual, results[0], rtol=0, atol=0) + + +def test_qsa_prefill_batch_invariance() -> None: + torch.manual_seed(42) + seq, heads, dim = 256, 3, 256 + q = torch.randn(seq, heads, dim, dtype=torch.bfloat16, device="cuda") + k = torch.randn(seq, 1, dim, dtype=q.dtype, device=q.device) + v = torch.randn_like(k) + positions = torch.arange(seq, device=q.device) + indices = positions.expand(seq, -1).to(torch.int32).contiguous() + indices.masked_fill_(positions[None, :] > positions[:, None], -1) + cu = torch.tensor([0, seq], dtype=torch.int32, device=q.device) + reference = sparse_gqa_fwd_interface_triton(q, k, v, seq, indices, cu, dim**-0.5) + for copies in (4, 8): + actual = sparse_gqa_fwd_interface_triton( + q.repeat(copies, 1, 1), + k.repeat(copies, 1, 1), + v.repeat(copies, 1, 1), + seq, + indices.repeat(copies, 1), + torch.arange(copies + 1, dtype=torch.int32, device=q.device) * seq, + dim**-0.5, + ) + torch.testing.assert_close(actual[:seq], reference, rtol=0, atol=0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/batch_invariant_ops/test_qwen_batch_invariance.py b/test/registered/unit/batch_invariant_ops/test_qwen_batch_invariance.py new file mode 100644 index 0000000000..33b5bf9311 --- /dev/null +++ b/test/registered/unit/batch_invariant_ops/test_qwen_batch_invariance.py @@ -0,0 +1,59 @@ +import sys +from collections.abc import Iterator + +import pytest +import torch + +from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add +from sglang.srt.batch_invariant_ops import ( + disable_batch_invariant_mode, + enable_batch_invariant_mode, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + + +@pytest.fixture(autouse=True) +def batch_invariant_mode() -> Iterator[None]: + enable_batch_invariant_mode() + try: + yield + finally: + disable_batch_invariant_mode() + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_mm_out_batch_invariance(dtype: torch.dtype) -> None: + torch.manual_seed(42) + a = torch.randn(257, 10240, dtype=dtype, device="cuda") + b = torch.randn(320, 10240, dtype=dtype, device="cuda").T + reference = torch.mm(a[:1], b) + for rows in (1, 16, 257): + output = torch.empty(rows, 320, dtype=dtype, device="cuda") + result = torch.mm(a[:rows], b, out=output) + assert result is output + torch.testing.assert_close(output[:1], reference, rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_shared_expert_gate_batch_invariance(dtype: torch.dtype) -> None: + torch.manual_seed(42) + hidden = torch.randn(256, 2560, dtype=dtype, device="cuda") + weight = torch.randn(2560, dtype=dtype, device="cuda") / 2560**0.5 + shared = torch.randn_like(hidden) + residual = torch.randn_like(hidden) + reference = residual.clone() + fused_gate_sigmoid_mul_add(hidden, weight, shared, reference) + for copies in (4, 8): + output = residual.repeat(copies, 1) + fused_gate_sigmoid_mul_add( + hidden.repeat(copies, 1), weight, shared.repeat(copies, 1), output + ) + torch.testing.assert_close(output[:256], reference, rtol=0, atol=0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))