[Fix] qwen3.8: Fix batch-invariant kernels for Qwen3.8 Flash-Next (#42087)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
wxzhoucs
2026-10-20 11:32:40 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 1140a8dfd9
commit e3205080ce
7 changed files with 206 additions and 2 deletions
@@ -3,6 +3,7 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.utils import is_hip
_is_hip = is_hip()
@@ -231,7 +232,7 @@ def _launch_fused_gate_sigmoid_mul(
),
}
if num_tokens >= 1024:
if num_tokens >= 1024 or is_batch_invariant_mode_enabled():
config["num_warps"] = min(config["num_warps"], 8)
use_pdl = is_arch_support_pdl()
@@ -602,6 +602,12 @@ def mm_batch_invariant(a, b):
return matmul_persistent(a, b)
def mm_out_batch_invariant(
a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor
) -> torch.Tensor:
return out.copy_(matmul_persistent(a, b))
def addmm_batch_invariant(bias, a, b):
return matmul_persistent(a, b, bias=bias)
@@ -1014,6 +1020,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
if not _is_npu:
# Register for detected device
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, dispatch_key)
_batch_invariant_LIB.impl(
"aten::_log_softmax", _log_softmax_batch_invariant, dispatch_key
@@ -8,6 +8,9 @@ import torch
import triton
import triton.language as tl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.utils import is_hip
def average_pool_qsa_keys(key_groups: torch.Tensor) -> torch.Tensor:
"""FP32-average complete key groups shaped ``[groups, ratio, kv_heads, dim]``."""
@@ -31,6 +34,19 @@ def qsa_fast_topk(
lengths = (row_ends - row_starts).to(device=logits.device, dtype=torch.int32)
starts = row_starts.to(device=logits.device, dtype=torch.int32)
if logits.is_cuda:
if is_batch_invariant_mode_enabled() and not is_hip():
from flashinfer import TopKTieBreak, top_k_ragged_transform
return top_k_ragged_transform(
logits,
torch.zeros_like(starts),
lengths,
topk,
deterministic=True,
tie_break=TopKTieBreak.SMALL,
dsa_graph_safe=True,
row_starts=starts,
)
if topk == 512:
# Prefer the JIT kernel: it ships with the sglang python package,
# so top-k 512 works regardless of the installed sgl_kernel version.
@@ -6,6 +6,7 @@ from typing import Tuple
import torch
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.layers.attention.qsa.kernel import (
average_pool_qsa_keys,
expand_qsa_block_indices,
@@ -501,7 +502,11 @@ class QSAIndexer(MultiPlatformOp):
) -> torch.Tensor:
# fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading
# their scores, and never reads past a row's length.
use_fast_topk = q.is_cuda and self.block_topk == 512
use_fast_topk = (
q.is_cuda
and self.block_topk == 512
and not is_batch_invariant_mode_enabled()
)
logits = qsa_mqa_decode(
q,
compressed_cache,
@@ -6,6 +6,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
_H20_CONFIGS = [
(32, (32, 8, 2)),
(64, (64, 8, 2)),
@@ -22,6 +24,8 @@ _L20_CONFIGS = [
def _get_best_config(total_q: int):
if is_batch_invariant_mode_enabled():
return (32, 4, 2)
table = _H20_CONFIGS if "H20" in torch.cuda.get_device_name(0) else _L20_CONFIGS
return next(cfg for limit, cfg in table if total_q <= limit)