[Fix] qwen3.8: Fix batch-invariant kernels for Qwen3.8 Flash-Next (#42087)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -231,7 +232,7 @@ def _launch_fused_gate_sigmoid_mul(
|
||||
),
|
||||
}
|
||||
|
||||
if num_tokens >= 1024:
|
||||
if num_tokens >= 1024 or is_batch_invariant_mode_enabled():
|
||||
config["num_warps"] = min(config["num_warps"], 8)
|
||||
|
||||
use_pdl = is_arch_support_pdl()
|
||||
|
||||
@@ -602,6 +602,12 @@ def mm_batch_invariant(a, b):
|
||||
return matmul_persistent(a, b)
|
||||
|
||||
|
||||
def mm_out_batch_invariant(
|
||||
a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return out.copy_(matmul_persistent(a, b))
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b):
|
||||
return matmul_persistent(a, b, bias=bias)
|
||||
|
||||
@@ -1014,6 +1020,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
|
||||
if not _is_npu:
|
||||
# Register for detected device
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, dispatch_key)
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::_log_softmax", _log_softmax_batch_invariant, dispatch_key
|
||||
|
||||
@@ -8,6 +8,9 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
|
||||
def average_pool_qsa_keys(key_groups: torch.Tensor) -> torch.Tensor:
|
||||
"""FP32-average complete key groups shaped ``[groups, ratio, kv_heads, dim]``."""
|
||||
@@ -31,6 +34,19 @@ def qsa_fast_topk(
|
||||
lengths = (row_ends - row_starts).to(device=logits.device, dtype=torch.int32)
|
||||
starts = row_starts.to(device=logits.device, dtype=torch.int32)
|
||||
if logits.is_cuda:
|
||||
if is_batch_invariant_mode_enabled() and not is_hip():
|
||||
from flashinfer import TopKTieBreak, top_k_ragged_transform
|
||||
|
||||
return top_k_ragged_transform(
|
||||
logits,
|
||||
torch.zeros_like(starts),
|
||||
lengths,
|
||||
topk,
|
||||
deterministic=True,
|
||||
tie_break=TopKTieBreak.SMALL,
|
||||
dsa_graph_safe=True,
|
||||
row_starts=starts,
|
||||
)
|
||||
if topk == 512:
|
||||
# Prefer the JIT kernel: it ships with the sglang python package,
|
||||
# so top-k 512 works regardless of the installed sgl_kernel version.
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.attention.qsa.kernel import (
|
||||
average_pool_qsa_keys,
|
||||
expand_qsa_block_indices,
|
||||
@@ -501,7 +502,11 @@ class QSAIndexer(MultiPlatformOp):
|
||||
) -> torch.Tensor:
|
||||
# fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading
|
||||
# their scores, and never reads past a row's length.
|
||||
use_fast_topk = q.is_cuda and self.block_topk == 512
|
||||
use_fast_topk = (
|
||||
q.is_cuda
|
||||
and self.block_topk == 512
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
)
|
||||
logits = qsa_mqa_decode(
|
||||
q,
|
||||
compressed_cache,
|
||||
|
||||
@@ -6,6 +6,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
|
||||
_H20_CONFIGS = [
|
||||
(32, (32, 8, 2)),
|
||||
(64, (64, 8, 2)),
|
||||
@@ -22,6 +24,8 @@ _L20_CONFIGS = [
|
||||
|
||||
|
||||
def _get_best_config(total_q: int):
|
||||
if is_batch_invariant_mode_enabled():
|
||||
return (32, 4, 2)
|
||||
table = _H20_CONFIGS if "H20" in torch.cuda.get_device_name(0) else _L20_CONFIGS
|
||||
return next(cfg for limit, cfg in table if total_q <= limit)
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.batch_invariant_ops import (
|
||||
disable_batch_invariant_mode,
|
||||
enable_batch_invariant_mode,
|
||||
)
|
||||
from sglang.srt.layers.attention.qsa import qsa_indexer as qsa_indexer_module
|
||||
from sglang.srt.layers.attention.qsa.kernel import qsa_fast_topk
|
||||
from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer
|
||||
from sglang.srt.layers.attention.qsa.sparse_attn import (
|
||||
sparse_gqa_fwd_interface_triton,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def batch_invariant_mode() -> Iterator[None]:
|
||||
enable_batch_invariant_mode()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
disable_batch_invariant_mode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", [512, 2048])
|
||||
def test_qsa_topk_ragged_ties_and_graph(topk: int) -> None:
|
||||
torch.manual_seed(42)
|
||||
lengths = [0, 1, 31, 511, 512, 513, 2047, 2048, 2049]
|
||||
logits = torch.randint(0, 32, (len(lengths), 4096), device="cuda").float()
|
||||
starts = torch.arange(len(lengths), dtype=torch.int32, device="cuda") * 37
|
||||
ends = starts + torch.tensor(lengths, dtype=torch.int32, device="cuda")
|
||||
expected = torch.full((len(lengths), topk), -1, dtype=torch.int32, device="cuda")
|
||||
for row, length in enumerate(lengths):
|
||||
start = row * 37
|
||||
order = torch.argsort(
|
||||
logits[row, start : start + length], descending=True, stable=True
|
||||
)
|
||||
count = min(length, topk)
|
||||
expected[row, :count] = order[:count].sort().values.to(torch.int32)
|
||||
actual = qsa_fast_topk(logits, starts, ends, topk)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = qsa_fast_topk(logits, starts, ends, topk)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(captured, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_qsa_decode_topk_repeatability(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
torch.manual_seed(42)
|
||||
logits = torch.randn(16, 2048, device="cuda").round()
|
||||
|
||||
def mock_mqa_decode(*args: object, min_scored_len: int) -> torch.Tensor:
|
||||
assert min_scored_len == 0
|
||||
return logits
|
||||
|
||||
monkeypatch.setattr(qsa_indexer_module, "qsa_mqa_decode", mock_mqa_decode)
|
||||
indexer = SimpleNamespace(block_topk=512)
|
||||
lengths = torch.full((16,), 2048, dtype=torch.int32, device="cuda")
|
||||
results = [
|
||||
QSAIndexer.select_decode_tokens(
|
||||
indexer,
|
||||
q=logits,
|
||||
compressed_cache=logits,
|
||||
compressed_page_table=lengths,
|
||||
compressed_lengths=lengths,
|
||||
max_model_len=2048,
|
||||
query_positions=lengths,
|
||||
sequence_lengths=lengths,
|
||||
defer_expansion=True,
|
||||
)
|
||||
for _ in range(4)
|
||||
]
|
||||
for actual in results[1:]:
|
||||
torch.testing.assert_close(actual, results[0], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_qsa_prefill_batch_invariance() -> None:
|
||||
torch.manual_seed(42)
|
||||
seq, heads, dim = 256, 3, 256
|
||||
q = torch.randn(seq, heads, dim, dtype=torch.bfloat16, device="cuda")
|
||||
k = torch.randn(seq, 1, dim, dtype=q.dtype, device=q.device)
|
||||
v = torch.randn_like(k)
|
||||
positions = torch.arange(seq, device=q.device)
|
||||
indices = positions.expand(seq, -1).to(torch.int32).contiguous()
|
||||
indices.masked_fill_(positions[None, :] > positions[:, None], -1)
|
||||
cu = torch.tensor([0, seq], dtype=torch.int32, device=q.device)
|
||||
reference = sparse_gqa_fwd_interface_triton(q, k, v, seq, indices, cu, dim**-0.5)
|
||||
for copies in (4, 8):
|
||||
actual = sparse_gqa_fwd_interface_triton(
|
||||
q.repeat(copies, 1, 1),
|
||||
k.repeat(copies, 1, 1),
|
||||
v.repeat(copies, 1, 1),
|
||||
seq,
|
||||
indices.repeat(copies, 1),
|
||||
torch.arange(copies + 1, dtype=torch.int32, device=q.device) * seq,
|
||||
dim**-0.5,
|
||||
)
|
||||
torch.testing.assert_close(actual[:seq], reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -0,0 +1,59 @@
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add
|
||||
from sglang.srt.batch_invariant_ops import (
|
||||
disable_batch_invariant_mode,
|
||||
enable_batch_invariant_mode,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def batch_invariant_mode() -> Iterator[None]:
|
||||
enable_batch_invariant_mode()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
disable_batch_invariant_mode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_mm_out_batch_invariance(dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(257, 10240, dtype=dtype, device="cuda")
|
||||
b = torch.randn(320, 10240, dtype=dtype, device="cuda").T
|
||||
reference = torch.mm(a[:1], b)
|
||||
for rows in (1, 16, 257):
|
||||
output = torch.empty(rows, 320, dtype=dtype, device="cuda")
|
||||
result = torch.mm(a[:rows], b, out=output)
|
||||
assert result is output
|
||||
torch.testing.assert_close(output[:1], reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_shared_expert_gate_batch_invariance(dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(42)
|
||||
hidden = torch.randn(256, 2560, dtype=dtype, device="cuda")
|
||||
weight = torch.randn(2560, dtype=dtype, device="cuda") / 2560**0.5
|
||||
shared = torch.randn_like(hidden)
|
||||
residual = torch.randn_like(hidden)
|
||||
reference = residual.clone()
|
||||
fused_gate_sigmoid_mul_add(hidden, weight, shared, reference)
|
||||
for copies in (4, 8):
|
||||
output = residual.repeat(copies, 1)
|
||||
fused_gate_sigmoid_mul_add(
|
||||
hidden.repeat(copies, 1), weight, shared.repeat(copies, 1), output
|
||||
)
|
||||
torch.testing.assert_close(output[:256], reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user