[Fix] qwen3.8: Fix batch-invariant kernels for Qwen3.8 Flash-Next (#42087)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -231,7 +232,7 @@ def _launch_fused_gate_sigmoid_mul(
|
||||
),
|
||||
}
|
||||
|
||||
if num_tokens >= 1024:
|
||||
if num_tokens >= 1024 or is_batch_invariant_mode_enabled():
|
||||
config["num_warps"] = min(config["num_warps"], 8)
|
||||
|
||||
use_pdl = is_arch_support_pdl()
|
||||
|
||||
@@ -602,6 +602,12 @@ def mm_batch_invariant(a, b):
|
||||
return matmul_persistent(a, b)
|
||||
|
||||
|
||||
def mm_out_batch_invariant(
|
||||
a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return out.copy_(matmul_persistent(a, b))
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b):
|
||||
return matmul_persistent(a, b, bias=bias)
|
||||
|
||||
@@ -1014,6 +1020,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
|
||||
if not _is_npu:
|
||||
# Register for detected device
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::_log_softmax", _log_softmax_batch_invariant, dispatch_key
|
||||
|
||||
@@ -8,6 +8,9 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
|
||||
def average_pool_qsa_keys(key_groups: torch.Tensor) -> torch.Tensor:
|
||||
"""FP32-average complete key groups shaped ``[groups, ratio, kv_heads, dim]``."""
|
||||
@@ -31,6 +34,19 @@ def qsa_fast_topk(
|
||||
lengths = (row_ends - row_starts).to(device=logits.device, dtype=torch.int32)
|
||||
starts = row_starts.to(device=logits.device, dtype=torch.int32)
|
||||
if logits.is_cuda:
|
||||
if is_batch_invariant_mode_enabled() and not is_hip():
|
||||
from flashinfer import TopKTieBreak, top_k_ragged_transform
|
||||
|
||||
return top_k_ragged_transform(
|
||||
logits,
|
||||
torch.zeros_like(starts),
|
||||
lengths,
|
||||
topk,
|
||||
deterministic=True,
|
||||
tie_break=TopKTieBreak.SMALL,
|
||||
dsa_graph_safe=True,
|
||||
row_starts=starts,
|
||||
)
|
||||
if topk == 512:
|
||||
# Prefer the JIT kernel: it ships with the sglang python package,
|
||||
# so top-k 512 works regardless of the installed sgl_kernel version.
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.attention.qsa.kernel import (
|
||||
average_pool_qsa_keys,
|
||||
expand_qsa_block_indices,
|
||||
@@ -501,7 +502,11 @@ class QSAIndexer(MultiPlatformOp):
|
||||
) -> torch.Tensor:
|
||||
# fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading
|
||||
# their scores, and never reads past a row's length.
|
||||
use_fast_topk = q.is_cuda and self.block_topk == 512
|
||||
use_fast_topk = (
|
||||
q.is_cuda
|
||||
and self.block_topk == 512
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
)
|
||||
logits = qsa_mqa_decode(
|
||||
q,
|
||||
compressed_cache,
|
||||
|
||||
@@ -6,6 +6,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
|
||||
_H20_CONFIGS = [
|
||||
(32, (32, 8, 2)),
|
||||
(64, (64, 8, 2)),
|
||||
@@ -22,6 +24,8 @@ _L20_CONFIGS = [
|
||||
|
||||
|
||||
def _get_best_config(total_q: int):
|
||||
if is_batch_invariant_mode_enabled():
|
||||
return (32, 4, 2)
|
||||
table = _H20_CONFIGS if "H20" in torch.cuda.get_device_name(0) else _L20_CONFIGS
|
||||
return next(cfg for limit, cfg in table if total_q <= limit)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user