[Fix] qwen3.8: Fix batch-invariant kernels for Qwen3.8 Flash-Next (#42087)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
wxzhoucs
2026-10-20 11:32:40 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 1140a8dfd9
commit e3205080ce
7 changed files with 206 additions and 2 deletions
@@ -3,6 +3,7 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.utils import is_hip
_is_hip = is_hip()
@@ -231,7 +232,7 @@ def _launch_fused_gate_sigmoid_mul(
),
}
if num_tokens >= 1024:
if num_tokens >= 1024 or is_batch_invariant_mode_enabled():
config["num_warps"] = min(config["num_warps"], 8)
use_pdl = is_arch_support_pdl()
@@ -602,6 +602,12 @@ def mm_batch_invariant(a, b):
return matmul_persistent(a, b)
def mm_out_batch_invariant(
a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor
) -> torch.Tensor:
return out.copy_(matmul_persistent(a, b))
def addmm_batch_invariant(bias, a, b):
return matmul_persistent(a, b, bias=bias)
@@ -1014,6 +1020,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
if not _is_npu:
# Register for detected device
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl(
"aten::_log_softmax", _log_softmax_batch_invariant, dispatch_key
@@ -8,6 +8,9 @@ import torch
import triton
import triton.language as tl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.utils import is_hip
def average_pool_qsa_keys(key_groups: torch.Tensor) -> torch.Tensor:
"""FP32-average complete key groups shaped ``[groups, ratio, kv_heads, dim]``."""
@@ -31,6 +34,19 @@ def qsa_fast_topk(
lengths = (row_ends - row_starts).to(device=logits.device, dtype=torch.int32)
starts = row_starts.to(device=logits.device, dtype=torch.int32)
if logits.is_cuda:
if is_batch_invariant_mode_enabled() and not is_hip():
from flashinfer import TopKTieBreak, top_k_ragged_transform
return top_k_ragged_transform(
logits,
torch.zeros_like(starts),
lengths,
topk,
deterministic=True,
tie_break=TopKTieBreak.SMALL,
dsa_graph_safe=True,
row_starts=starts,
)
if topk == 512:
# Prefer the JIT kernel: it ships with the sglang python package,
# so top-k 512 works regardless of the installed sgl_kernel version.
@@ -6,6 +6,7 @@ from typing import Tuple
import torch
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.attention.qsa.kernel import (
average_pool_qsa_keys,
expand_qsa_block_indices,
@@ -501,7 +502,11 @@ class QSAIndexer(MultiPlatformOp):
) -> torch.Tensor:
# fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading
# their scores, and never reads past a row's length.
use_fast_topk = q.is_cuda and self.block_topk == 512
use_fast_topk = (
q.is_cuda
and self.block_topk == 512
and not is_batch_invariant_mode_enabled()
)
logits = qsa_mqa_decode(
q,
compressed_cache,
@@ -6,6 +6,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
_H20_CONFIGS = [
(32, (32, 8, 2)),
(64, (64, 8, 2)),
@@ -22,6 +24,8 @@ _L20_CONFIGS = [
def _get_best_config(total_q: int):
if is_batch_invariant_mode_enabled():
return (32, 4, 2)
table = _H20_CONFIGS if "H20" in torch.cuda.get_device_name(0) else _L20_CONFIGS
return next(cfg for limit, cfg in table if total_q <= limit)
@@ -0,0 +1,112 @@
import sys
from collections.abc import Iterator
from types import SimpleNamespace
import pytest
import torch
from sglang.srt.batch_invariant_ops import (
disable_batch_invariant_mode,
enable_batch_invariant_mode,
)
from sglang.srt.layers.attention.qsa import qsa_indexer as qsa_indexer_module
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer
from sglang.srt.layers.attention.qsa.sparse_attn import (
sparse_gqa_fwd_interface_triton,
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=30, stage="nightly", runner_config="1-gpu-large")
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.fixture(autouse=True)
def batch_invariant_mode() -> Iterator[None]:
enable_batch_invariant_mode()
try:
yield
finally:
disable_batch_invariant_mode()
@pytest.mark.parametrize("topk", [512, 2048])
def test_qsa_topk_ragged_ties_and_graph(topk: int) -> None:
torch.manual_seed(42)
lengths = [0, 1, 31, 511, 512, 513, 2047, 2048, 2049]
logits = torch.randint(0, 32, (len(lengths), 4096), device="cuda").float()
starts = torch.arange(len(lengths), dtype=torch.int32, device="cuda") * 37
ends = starts + torch.tensor(lengths, dtype=torch.int32, device="cuda")
expected = torch.full((len(lengths), topk), -1, dtype=torch.int32, device="cuda")
for row, length in enumerate(lengths):
start = row * 37
order = torch.argsort(
logits[row, start : start + length], descending=True, stable=True
)
count = min(length, topk)
expected[row, :count] = order[:count].sort().values.to(torch.int32)
actual = qsa_fast_topk(logits, starts, ends, topk)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = qsa_fast_topk(logits, starts, ends, topk)
graph.replay()
torch.testing.assert_close(captured, expected, rtol=0, atol=0)
def test_qsa_decode_topk_repeatability(monkeypatch: pytest.MonkeyPatch) -> None:
torch.manual_seed(42)
logits = torch.randn(16, 2048, device="cuda").round()
def mock_mqa_decode(*args: object, min_scored_len: int) -> torch.Tensor:
assert min_scored_len == 0
return logits
monkeypatch.setattr(qsa_indexer_module, "qsa_mqa_decode", mock_mqa_decode)
indexer = SimpleNamespace(block_topk=512)
lengths = torch.full((16,), 2048, dtype=torch.int32, device="cuda")
results = [
QSAIndexer.select_decode_tokens(
indexer,
q=logits,
compressed_cache=logits,
compressed_page_table=lengths,
compressed_lengths=lengths,
max_model_len=2048,
query_positions=lengths,
sequence_lengths=lengths,
defer_expansion=True,
)
for _ in range(4)
]
for actual in results[1:]:
torch.testing.assert_close(actual, results[0], rtol=0, atol=0)
def test_qsa_prefill_batch_invariance() -> None:
torch.manual_seed(42)
seq, heads, dim = 256, 3, 256
q = torch.randn(seq, heads, dim, dtype=torch.bfloat16, device="cuda")
k = torch.randn(seq, 1, dim, dtype=q.dtype, device=q.device)
v = torch.randn_like(k)
positions = torch.arange(seq, device=q.device)
indices = positions.expand(seq, -1).to(torch.int32).contiguous()
indices.masked_fill_(positions[None, :] > positions[:, None], -1)
cu = torch.tensor([0, seq], dtype=torch.int32, device=q.device)
reference = sparse_gqa_fwd_interface_triton(q, k, v, seq, indices, cu, dim**-0.5)
for copies in (4, 8):
actual = sparse_gqa_fwd_interface_triton(
q.repeat(copies, 1, 1),
k.repeat(copies, 1, 1),
v.repeat(copies, 1, 1),
seq,
indices.repeat(copies, 1),
torch.arange(copies + 1, dtype=torch.int32, device=q.device) * seq,
dim**-0.5,
)
torch.testing.assert_close(actual[:seq], reference, rtol=0, atol=0)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -0,0 +1,59 @@
import sys
from collections.abc import Iterator
import pytest
import torch
from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add
from sglang.srt.batch_invariant_ops import (
disable_batch_invariant_mode,
enable_batch_invariant_mode,
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.fixture(autouse=True)
def batch_invariant_mode() -> Iterator[None]:
enable_batch_invariant_mode()
try:
yield
finally:
disable_batch_invariant_mode()
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_mm_out_batch_invariance(dtype: torch.dtype) -> None:
torch.manual_seed(42)
a = torch.randn(257, 10240, dtype=dtype, device="cuda")
b = torch.randn(320, 10240, dtype=dtype, device="cuda").T
reference = torch.mm(a[:1], b)
for rows in (1, 16, 257):
output = torch.empty(rows, 320, dtype=dtype, device="cuda")
result = torch.mm(a[:rows], b, out=output)
assert result is output
torch.testing.assert_close(output[:1], reference, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_shared_expert_gate_batch_invariance(dtype: torch.dtype) -> None:
torch.manual_seed(42)
hidden = torch.randn(256, 2560, dtype=dtype, device="cuda")
weight = torch.randn(2560, dtype=dtype, device="cuda") / 2560**0.5
shared = torch.randn_like(hidden)
residual = torch.randn_like(hidden)
reference = residual.clone()
fused_gate_sigmoid_mul_add(hidden, weight, shared, reference)
for copies in (4, 8):
output = residual.repeat(copies, 1)
fused_gate_sigmoid_mul_add(
hidden.repeat(copies, 1), weight, shared.repeat(copies, 1), output
)
torch.testing.assert_close(output[:256], reference, rtol=0, atol=0)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))