[minimax-m3] Split 4/4: model + VL + glue + function-call + fp8 quant + generic infra (#28715)
Co-authored-by: Xinyuan Tong <xinyuan-tong@users.noreply.github.com> Co-authored-by: zijiexia <37504505+zijiexia@users.noreply.github.com>
This commit is contained in:
Xinyuan Tong
co-authored by
Xinyuan Tong
zijiexia
Xinyuan Tong
zijiexia
parent
e3ceccf781
commit
0663ebc783
@@ -154,9 +154,9 @@ CONFIGS_MOE = list(
|
||||
# ---- Final configs ----
|
||||
CONFIGS = CONFIGS_GEMM + CONFIGS_MOE
|
||||
|
||||
LINE_VALS = ["triton", "sglang"]
|
||||
LINE_NAMES = ["Triton (Inaccurate)", "SGL Kernel"]
|
||||
STYLES = [("blue", "-"), ("green", "-")]
|
||||
LINE_VALS = ["triton", "aot_v2", "sglang"]
|
||||
LINE_NAMES = ["Triton (Inaccurate)", "AOT v2 (sgl-kernel)", "JIT (this repo)"]
|
||||
STYLES = [("blue", "-"), ("red", "-"), ("green", "-")]
|
||||
|
||||
|
||||
def _flatten_to_2d(t: torch.Tensor) -> torch.Tensor:
|
||||
@@ -171,6 +171,7 @@ def _make_sglang_bench_fn(
|
||||
group_size: int,
|
||||
dst_dtype: torch.dtype,
|
||||
flags: dict,
|
||||
provider: str = "sglang",
|
||||
):
|
||||
"""
|
||||
Adapter that pre-allocates output tensors and returns a zero-arg callable
|
||||
@@ -213,17 +214,37 @@ def _make_sglang_bench_fn(
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
|
||||
def _run():
|
||||
sglang_per_token_group_quant_8bit(
|
||||
input=x_input,
|
||||
output_q=output_q,
|
||||
output_s=output_s,
|
||||
group_size=group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
if provider == "aot_v2":
|
||||
from sgl_kernel import sgl_per_token_group_quant_8bit as aot_quant
|
||||
|
||||
def _run():
|
||||
aot_quant(
|
||||
x_input,
|
||||
output_q,
|
||||
output_s,
|
||||
group_size,
|
||||
1e-10,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
scale_ue8m0,
|
||||
False, # fuse_silu_and_mul (already applied to x_input)
|
||||
None, # masked_m (flattened to 2D)
|
||||
enable_v2=True,
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def _run():
|
||||
sglang_per_token_group_quant_8bit(
|
||||
input=x_input,
|
||||
output_q=output_q,
|
||||
output_s=output_s,
|
||||
group_size=group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
|
||||
return _run
|
||||
|
||||
@@ -270,13 +291,14 @@ def benchmark(
|
||||
dst_dtype=dst_dtype,
|
||||
**{k: v for k, v in flags.items() if k not in ["masked_layout_mode"]},
|
||||
)
|
||||
elif provider == "sglang":
|
||||
elif provider in ("sglang", "aot_v2"):
|
||||
kernel_names = "per_token_group_quant_8bit_kernel"
|
||||
bench_fn = _make_sglang_bench_fn(
|
||||
x=x,
|
||||
group_size=group_size,
|
||||
dst_dtype=dst_dtype,
|
||||
flags=flags,
|
||||
provider=provider,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown provider: {provider}")
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import torch
|
||||
|
||||
from sglang.jit_kernel.benchmark import marker
|
||||
from sglang.srt.layers.moe.ep_moe.kernels import (
|
||||
post_reorder_deepgemm,
|
||||
post_reorder_triton_kernel,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=8, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
|
||||
)
|
||||
|
||||
HIDDEN = 6144
|
||||
NUM_EXPERTS = 129
|
||||
TOP_K = 5
|
||||
RSF = 2.0
|
||||
|
||||
|
||||
def _build(num_tokens):
|
||||
m_max = (num_tokens // 256 + 1) * 256
|
||||
down_output = torch.randn(
|
||||
NUM_EXPERTS * m_max, HIDDEN, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0, NUM_EXPERTS, (num_tokens, TOP_K), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
src2dst = (
|
||||
topk_ids.long() * m_max
|
||||
+ torch.randint(0, m_max, (num_tokens, TOP_K), device="cuda")
|
||||
).to(torch.int32)
|
||||
topk_weights = torch.rand(num_tokens, TOP_K, dtype=torch.float32, device="cuda")
|
||||
return down_output, src2dst, topk_ids, topk_weights
|
||||
|
||||
|
||||
def _new(down_output, src2dst, topk_ids, topk_weights):
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out = torch.empty(num_tokens, HIDDEN, dtype=torch.bfloat16, device="cuda")
|
||||
post_reorder_deepgemm(
|
||||
down_output,
|
||||
out,
|
||||
src2dst,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
TOP_K,
|
||||
num_tokens,
|
||||
HIDDEN,
|
||||
RSF,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _old(down_output, src2dst, topk_ids, topk_weights):
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out = torch.empty(num_tokens, HIDDEN, dtype=torch.bfloat16, device="cuda")
|
||||
post_reorder_triton_kernel[(num_tokens,)](
|
||||
down_output, out, src2dst, topk_ids, topk_weights, TOP_K, HIDDEN, BLOCK_SIZE=512
|
||||
)
|
||||
out *= RSF
|
||||
return out
|
||||
|
||||
|
||||
FN_MAP = {"fused": _new, "legacy": _old}
|
||||
|
||||
|
||||
@marker.parametrize("num_tokens", [1, 8, 64, 256, 1024, 4096, 16384], [64, 4096])
|
||||
@marker.benchmark("impl", ["fused", "legacy"])
|
||||
def benchmark(num_tokens: int, impl: str):
|
||||
down_output, src2dst, topk_ids, topk_weights = _build(num_tokens)
|
||||
return marker.do_bench(
|
||||
FN_MAP[impl],
|
||||
input_args=(down_output, src2dst, topk_ids, topk_weights),
|
||||
graph_clone_args=(0,),
|
||||
memory_args=None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run()
|
||||
@@ -0,0 +1,69 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
dev = "cuda"
|
||||
|
||||
|
||||
def _pack_weight_scale(scale_u8: torch.Tensor) -> torch.Tensor:
|
||||
from sglang.srt.layers.deep_gemm_wrapper.configurer import DEEPGEMM_SCALE_UE8M0
|
||||
|
||||
n, kk = scale_u8.shape
|
||||
scale_fp32 = (
|
||||
(scale_u8.contiguous().view(-1).to(torch.int32) << 23)
|
||||
.view(torch.float32)
|
||||
.view(n, kk)
|
||||
)
|
||||
if DEEPGEMM_SCALE_UE8M0:
|
||||
import deep_gemm.utils.layout
|
||||
|
||||
return deep_gemm.utils.layout.get_mn_major_tma_aligned_packed_ue8m0_tensor(
|
||||
scale_fp32
|
||||
)
|
||||
return scale_fp32
|
||||
|
||||
|
||||
@pytest.mark.parametrize("T", [1, 7, 64, 256, 1024])
|
||||
@pytest.mark.parametrize("N1,N2,K", [(1280, 256, 6144), (1024, 128, 2048)])
|
||||
def test_fused_equals_separate(T, N1, N2, K):
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_w8a8_mxfp8_linear_with_fallback as mxfp8_linear,
|
||||
)
|
||||
|
||||
torch.manual_seed(T * 7 + N1 + K)
|
||||
G = K // 32
|
||||
|
||||
def rand_w(n):
|
||||
return (torch.randn(n, K, device=dev) * 0.2).to(torch.float8_e4m3fn)
|
||||
|
||||
def rand_s(n):
|
||||
# mid-range UE8M0 exponents (~2^-3..2^3) to avoid inf/zero blowups.
|
||||
return torch.randint(124, 131, (n, G), device=dev, dtype=torch.uint8)
|
||||
|
||||
w1, w2 = rand_w(N1), rand_w(N2)
|
||||
s1, s2 = rand_s(N1), rand_s(N2)
|
||||
x = torch.randn(T, K, dtype=torch.bfloat16, device=dev)
|
||||
|
||||
s1p, s2p = _pack_weight_scale(s1), _pack_weight_scale(s2)
|
||||
out1 = mxfp8_linear(x, w1, s1p, weight_scale_fallback=s1)
|
||||
out2 = mxfp8_linear(x, w2, s2p, weight_scale_fallback=s2)
|
||||
ref = torch.cat([out1, out2], dim=-1)
|
||||
|
||||
w = torch.cat([w1, w2], dim=0).contiguous()
|
||||
s = torch.cat([s1, s2], dim=0).contiguous()
|
||||
sp = _pack_weight_scale(s)
|
||||
fused = mxfp8_linear(x, w, sp, weight_scale_fallback=s)
|
||||
|
||||
assert fused.shape == ref.shape
|
||||
assert torch.equal(
|
||||
fused, ref
|
||||
), f"max abs diff {(fused.float() - ref.float()).abs().max().item()}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v", "-s"]))
|
||||
@@ -0,0 +1,84 @@
|
||||
import random
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.jit_kernel.minimax_quant_ue8m0 import (
|
||||
per_token_quant_fp8_ue8m0,
|
||||
per_token_quant_fp8_ue8m0_scatter,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.kernels import fill_gateup_input_triton_kernel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
dev = "cuda"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 256])
|
||||
@pytest.mark.parametrize("topk", [4, 5, 8])
|
||||
@pytest.mark.parametrize("hidden,group", [(6144, 32), (2048, 32), (4096, 128)])
|
||||
def test_quant_scatter_matches_quant_plus_fill(num_tokens, topk, hidden, group):
|
||||
arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device())
|
||||
if arch_major <= 9:
|
||||
pytest.skip("UE8M0 fusion is Blackwell-only")
|
||||
|
||||
E = 129 # 128 routed + 1 fused shared
|
||||
G4 = (hidden // group) // 4
|
||||
m_max = (num_tokens // 256 + 1) * 256
|
||||
torch.manual_seed(num_tokens * 91 + topk + hidden)
|
||||
random.seed(num_tokens)
|
||||
|
||||
x = (torch.randn(num_tokens, hidden, device=dev, dtype=torch.bfloat16)) * 4.0
|
||||
|
||||
tids = torch.empty(num_tokens, topk, dtype=torch.int32, device=dev)
|
||||
tids_cpu = torch.empty(num_tokens, topk, dtype=torch.int32)
|
||||
s2d = [0] * (num_tokens * topk)
|
||||
cur = [0] * E
|
||||
for t in range(num_tokens):
|
||||
for j, e in enumerate(random.sample(range(E), topk)):
|
||||
tids_cpu[t, j] = e
|
||||
s2d[t * topk + j] = e * m_max + cur[e]
|
||||
cur[e] += 1
|
||||
tids.copy_(tids_cpu)
|
||||
s2d = torch.tensor(s2d, dtype=torch.int32, device=dev)
|
||||
|
||||
x_q, x_sf = per_token_quant_fp8_ue8m0(x, group)
|
||||
gi_ref = torch.zeros(E, m_max, hidden, device=dev, dtype=torch.float8_e4m3fn)
|
||||
gs_ref = torch.zeros(E, G4, m_max, device=dev, dtype=torch.int32)
|
||||
fill_gateup_input_triton_kernel[(num_tokens,)](
|
||||
x_q,
|
||||
x_sf,
|
||||
gi_ref,
|
||||
gs_ref,
|
||||
s2d,
|
||||
tids,
|
||||
topk,
|
||||
hidden,
|
||||
G4,
|
||||
m_max,
|
||||
x_sf.stride(0),
|
||||
x_sf.stride(1),
|
||||
BLOCK_SIZE=1024,
|
||||
SCALE_MN_MAJOR=True,
|
||||
)
|
||||
|
||||
gi_new = torch.zeros(E, m_max, hidden, device=dev, dtype=torch.float8_e4m3fn)
|
||||
gs_new = torch.zeros(E, G4, m_max, device=dev, dtype=torch.int32)
|
||||
per_token_quant_fp8_ue8m0_scatter(x, gi_new, gs_new, s2d, tids, topk, m_max, group)
|
||||
|
||||
for t in range(num_tokens):
|
||||
for j in range(topk):
|
||||
e = int(tids_cpu[t, j])
|
||||
m = int(s2d[t * topk + j]) % m_max
|
||||
assert torch.equal(
|
||||
gi_new[e, m].view(torch.uint8), gi_ref[e, m].view(torch.uint8)
|
||||
), f"fp8 mismatch token={t} slot={j} expert={e}"
|
||||
assert torch.equal(
|
||||
gs_new[e, :, m], gs_ref[e, :, m]
|
||||
), f"scale mismatch token={t} slot={j} expert={e}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v", "-x"]))
|
||||
@@ -17,11 +17,17 @@ register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required", allow_module_level=True)
|
||||
|
||||
from sgl_kernel import ( # noqa: E402
|
||||
sgl_per_token_group_quant_8bit as aot_per_token_group_quant_8bit,
|
||||
)
|
||||
from sgl_kernel.test_utils import ( # noqa: E402
|
||||
assert_all_close_or_tiny_diff,
|
||||
create_per_token_group_quant_test_data,
|
||||
)
|
||||
|
||||
from sglang.jit_kernel.per_token_group_quant_8bit import ( # noqa: E402
|
||||
per_token_group_quant_8bit as jit_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_kernel import ( # noqa: E402
|
||||
create_per_token_group_quant_fp8_output_scale,
|
||||
)
|
||||
@@ -224,5 +230,104 @@ def test_per_token_group_quant_with_column_major(
|
||||
raise
|
||||
|
||||
|
||||
LAYOUTS = [
|
||||
(False, False, False),
|
||||
(True, False, False),
|
||||
(True, True, False),
|
||||
(True, True, True),
|
||||
]
|
||||
|
||||
CONFIGS = list(
|
||||
itertools.product(
|
||||
[1, 4, 16, 64, 127, 128, 512, 1024, 4096, 8192],
|
||||
[512, 1536, 2048, 4096, 6144, 7168, 16384],
|
||||
[16, 32, 64, 128],
|
||||
LAYOUTS,
|
||||
[fp8_type_],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens, hidden_dim, group_size, layout, dst_dtype", CONFIGS
|
||||
)
|
||||
def test_jit_matches_aot_v2_byte_identical(
|
||||
num_tokens, hidden_dim, group_size, layout, dst_dtype
|
||||
):
|
||||
column_major_scales, scale_tma_aligned, scale_ue8m0 = layout
|
||||
|
||||
arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device())
|
||||
if scale_ue8m0 and arch_major <= 9:
|
||||
pytest.skip("UE8M0 fusion is Blackwell-only")
|
||||
if hidden_dim % group_size != 0:
|
||||
pytest.skip("hidden_dim must be divisible by group_size")
|
||||
|
||||
torch.manual_seed(num_tokens * 131 + hidden_dim + group_size)
|
||||
x = (torch.randn(num_tokens, hidden_dim, device="cuda", dtype=torch.bfloat16)) * 3.0
|
||||
|
||||
fp8_max = torch.finfo(dst_dtype).max
|
||||
fp8_min = -fp8_max
|
||||
|
||||
def _alloc():
|
||||
q = torch.empty_like(x, dtype=dst_dtype)
|
||||
s = create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=x.shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
return q, s
|
||||
|
||||
q_aot, s_aot = _alloc()
|
||||
aot_per_token_group_quant_8bit(
|
||||
x,
|
||||
q_aot,
|
||||
s_aot,
|
||||
group_size,
|
||||
1e-10,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
scale_ue8m0,
|
||||
False,
|
||||
None,
|
||||
enable_v2=True,
|
||||
)
|
||||
|
||||
q_jit, s_jit = _alloc()
|
||||
jit_per_token_group_quant_8bit(
|
||||
x, q_jit, s_jit, group_size, 1e-10, fp8_min, fp8_max, scale_ue8m0=scale_ue8m0
|
||||
)
|
||||
|
||||
# AOT v2 uses -use_fast_math reciprocal; this JIT uses precise division, so an
|
||||
# exact fp8 midpoint can round to an adjacent code (1-ULP, JIT more accurate).
|
||||
qj = q_jit.view(torch.uint8)
|
||||
qa = q_aot.view(torch.uint8)
|
||||
if not torch.equal(qj, qa):
|
||||
mism = qj != qa
|
||||
bj = qj[mism].to(torch.int16)
|
||||
ba = qa[mism].to(torch.int16)
|
||||
same_sign = (bj & 0x80) == (ba & 0x80)
|
||||
one_ulp = (bj - ba).abs() == 1
|
||||
assert bool(
|
||||
(same_sign & one_ulp).all()
|
||||
), f"q mismatch > 1 fp8 ULP {num_tokens=} {hidden_dim=} {group_size=} {layout=}"
|
||||
assert mism.float().mean() < 0.01, (
|
||||
f"too many fp8 ties ({int(mism.sum())}/{mism.numel()}) "
|
||||
f"{num_tokens=} {hidden_dim=} {group_size=} {layout=}"
|
||||
)
|
||||
|
||||
if scale_ue8m0:
|
||||
assert torch.equal(
|
||||
s_jit[:num_tokens].reshape(num_tokens, -1).view(torch.int32),
|
||||
s_aot[:num_tokens].reshape(num_tokens, -1).view(torch.int32),
|
||||
), f"ue8m0 scale mismatch {num_tokens=} {hidden_dim=} {group_size=}"
|
||||
else:
|
||||
assert torch.equal(
|
||||
s_jit[:num_tokens].float(), s_aot[:num_tokens].float()
|
||||
), f"float scale mismatch {num_tokens=} {hidden_dim=} {group_size=}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v", "-s"]))
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.ep_moe.kernels import (
|
||||
post_reorder_deepgemm,
|
||||
post_reorder_triton_kernel,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
dev = "cuda"
|
||||
|
||||
|
||||
def _build(
|
||||
num_tokens,
|
||||
hidden,
|
||||
num_routed=128,
|
||||
top_k_routed=4,
|
||||
with_shared=True,
|
||||
pad_frac=0.0,
|
||||
seed=0,
|
||||
):
|
||||
num_experts = num_routed + (1 if with_shared else 0)
|
||||
top_k = top_k_routed + (1 if with_shared else 0)
|
||||
m_max = (num_tokens // 256 + 1) * 256
|
||||
g = torch.Generator(device="cpu").manual_seed(seed)
|
||||
down_output = torch.randn(
|
||||
num_experts * m_max, hidden, dtype=torch.bfloat16, device=dev
|
||||
)
|
||||
topk_ids = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=dev)
|
||||
src2dst = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=dev)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.float32, device=dev)
|
||||
counts = [0] * num_experts
|
||||
for t in range(num_tokens):
|
||||
experts = torch.randperm(num_routed, generator=g)[:top_k_routed].tolist()
|
||||
if with_shared:
|
||||
experts = experts + [num_routed]
|
||||
for slot, e in enumerate(experts):
|
||||
if (
|
||||
pad_frac > 0
|
||||
and slot < top_k - 1
|
||||
and torch.rand(1, generator=g).item() < pad_frac
|
||||
):
|
||||
continue
|
||||
src2dst[t, slot] = e * m_max + counts[e]
|
||||
counts[e] += 1
|
||||
topk_ids[t, slot] = e
|
||||
return down_output, src2dst, topk_ids, topk_weights, top_k
|
||||
|
||||
|
||||
def _ref(down_output, src2dst, topk_ids, topk_weights, top_k, hidden, rsf):
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out = torch.zeros(num_tokens, hidden, dtype=torch.float32, device=dev)
|
||||
do = down_output.float()
|
||||
for slot in range(top_k):
|
||||
valid = topk_ids[:, slot] >= 0
|
||||
dst = src2dst[:, slot].long().clamp_min(0)
|
||||
out[valid] += (do[dst] * topk_weights[:, slot, None])[valid]
|
||||
return out * rsf
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 128, 1024, 4096])
|
||||
@pytest.mark.parametrize("pad_frac", [0.0, 0.3])
|
||||
@pytest.mark.parametrize("rsf", [1.0, 2.0])
|
||||
def test_post_reorder_deepgemm(num_tokens, pad_frac, rsf):
|
||||
hidden = 6144
|
||||
do, s2d, tids, tw, tk = _build(
|
||||
num_tokens, hidden, pad_frac=pad_frac, seed=num_tokens
|
||||
)
|
||||
ref = _ref(do, s2d, tids, tw, tk, hidden, rsf)
|
||||
|
||||
new = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=dev)
|
||||
post_reorder_deepgemm(do, new, s2d, tids, tw, tk, num_tokens, hidden, rsf)
|
||||
|
||||
old = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=dev)
|
||||
post_reorder_triton_kernel[(num_tokens,)](
|
||||
do, old, s2d, tids, tw, tk, hidden, BLOCK_SIZE=512
|
||||
)
|
||||
old *= rsf
|
||||
|
||||
torch.testing.assert_close(new.float(), ref, rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(new.float(), old.float(), rtol=5e-2, atol=0.5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v", "-s"]))
|
||||
@@ -0,0 +1,539 @@
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import Function, Tool
|
||||
from sglang.srt.function_call.minimax_m3 import MINIMAX_NS_TOKEN, MinimaxM3Detector
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=7, suite="base-a-test-cpu")
|
||||
|
||||
NS = MINIMAX_NS_TOKEN
|
||||
|
||||
|
||||
def _make_tools():
|
||||
return [
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="get_current_date",
|
||||
description="Get the current date",
|
||||
parameters={},
|
||||
),
|
||||
),
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="get_weather",
|
||||
description="Get weather information",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string", "description": "City name"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
),
|
||||
),
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="search",
|
||||
description="Search the web",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"count": {"type": "integer"},
|
||||
"ratio": {"type": "number"},
|
||||
"verbose": {"type": "boolean"},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
),
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="create_event",
|
||||
description="Create a calendar event",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"location": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"zip": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="add_note",
|
||||
description="Add a free-form note",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"note": {"type": "string"}},
|
||||
},
|
||||
),
|
||||
),
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name="configure",
|
||||
description="Configure runtime options",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["none", "low", "high"],
|
||||
},
|
||||
"optional": {"type": ["string", "null"]},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _wire(*lines):
|
||||
return "".join(NS + line for line in lines)
|
||||
|
||||
|
||||
def _segments(*lines):
|
||||
return [NS + line for line in lines]
|
||||
|
||||
|
||||
def _collect_streamed_tool_calls(all_calls):
|
||||
tools = {}
|
||||
for c in all_calls:
|
||||
idx = c.tool_index
|
||||
if idx not in tools:
|
||||
tools[idx] = {"name": c.name or "", "parameters": c.parameters or ""}
|
||||
else:
|
||||
if c.name:
|
||||
tools[idx]["name"] += c.name
|
||||
if c.parameters:
|
||||
tools[idx]["parameters"] += c.parameters
|
||||
return [tools[i] for i in sorted(tools.keys())]
|
||||
|
||||
|
||||
def _stream_segments(segments, tools):
|
||||
detector = MinimaxM3Detector()
|
||||
all_calls = []
|
||||
for seg in _segments(*segments):
|
||||
all_calls.extend(detector.parse_streaming_increment(seg, tools).calls)
|
||||
collected = _collect_streamed_tool_calls(all_calls)
|
||||
return [{"name": c["name"], "args": json.loads(c["parameters"])} for c in collected]
|
||||
|
||||
|
||||
def _parse_segments(segments, tools):
|
||||
detector = MinimaxM3Detector()
|
||||
result = detector.detect_and_parse(_wire(*segments), tools)
|
||||
return [
|
||||
{"name": c.name, "args": json.loads(c.parameters)} for c in result.calls
|
||||
], result.normal_text
|
||||
|
||||
|
||||
class TestMinimaxM3HasToolCall(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.detector = MinimaxM3Detector()
|
||||
|
||||
def test_has_tool_call_true(self):
|
||||
text = _wire(
|
||||
"<tool_call>",
|
||||
'<invoke name="get_current_date">',
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
self.assertTrue(self.detector.has_tool_call(text))
|
||||
|
||||
def test_has_tool_call_false(self):
|
||||
self.assertFalse(
|
||||
self.detector.has_tool_call("The weather in Beijing is sunny.")
|
||||
)
|
||||
|
||||
|
||||
class TestMinimaxM3DetectAndParse(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.tools = _make_tools()
|
||||
|
||||
def test_no_tool_call(self):
|
||||
text = "This is a plain response with no tool call."
|
||||
calls, normal = _parse_segments_text(text, self.tools)
|
||||
self.assertEqual(len(calls), 0)
|
||||
self.assertEqual(normal, text)
|
||||
|
||||
def test_single_tool_call(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["name"], "get_weather")
|
||||
self.assertEqual(calls[0]["args"], {"city": "Beijing"})
|
||||
|
||||
def test_zero_arg_tool_call(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_current_date">',
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["name"], "get_current_date")
|
||||
self.assertEqual(calls[0]["args"], {})
|
||||
|
||||
def test_multiple_tool_calls_separate_blocks(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Tokyo",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[0]["args"]["city"], "Beijing")
|
||||
self.assertEqual(calls[1]["args"]["city"], "Tokyo")
|
||||
|
||||
def test_multiple_invokes_one_block(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Tokyo",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[0]["args"]["city"], "Beijing")
|
||||
self.assertEqual(calls[1]["args"]["city"], "Tokyo")
|
||||
|
||||
def test_content_before_tool_call(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Paris",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
detector = MinimaxM3Detector()
|
||||
text = "Let me check the weather." + _wire(*segments)
|
||||
result = detector.detect_and_parse(text, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.normal_text, "Let me check the weather.")
|
||||
|
||||
def test_typed_scalars(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="search">',
|
||||
"<query>pizza",
|
||||
"</query>",
|
||||
"<count>7",
|
||||
"</count>",
|
||||
"<ratio>0.5",
|
||||
"</ratio>",
|
||||
"<verbose>true",
|
||||
"</verbose>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
args = calls[0]["args"]
|
||||
self.assertEqual(args["query"], "pizza")
|
||||
self.assertEqual(args["count"], 7)
|
||||
self.assertIsInstance(args["count"], int)
|
||||
self.assertAlmostEqual(args["ratio"], 0.5)
|
||||
self.assertIs(args["verbose"], True)
|
||||
|
||||
def test_nested_object_and_array(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="create_event">',
|
||||
"<title>Standup",
|
||||
"</title>",
|
||||
"<location>",
|
||||
"<city>NYC",
|
||||
"</city>",
|
||||
"<zip>10001",
|
||||
"</zip>",
|
||||
"</location>",
|
||||
"<tags>",
|
||||
"<item>red",
|
||||
"</item>",
|
||||
"<item>blue",
|
||||
"</item>",
|
||||
"</tags>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(
|
||||
calls[0]["args"],
|
||||
{
|
||||
"title": "Standup",
|
||||
"location": {"city": "NYC", "zip": 10001},
|
||||
"tags": ["red", "blue"],
|
||||
},
|
||||
)
|
||||
|
||||
def test_string_with_special_characters(self):
|
||||
value = 'a "quoted" word with \\ backslash and\nnewline'
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="add_note">',
|
||||
"<note>" + value,
|
||||
"</note>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(calls[0]["args"], {"note": value})
|
||||
|
||||
|
||||
class TestMinimaxM3NoneNullRegression(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.tools = _make_tools()
|
||||
|
||||
def _configure_segments(self, param, value):
|
||||
return (
|
||||
"<tool_call>",
|
||||
'<invoke name="configure">',
|
||||
"<{}>{}".format(param, value),
|
||||
"</{}>".format(param),
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
|
||||
def test_plain_string_none_not_coerced(self):
|
||||
for value in ("none", "nil", "null"):
|
||||
with self.subTest(value=value):
|
||||
segments = self._configure_segments("mode", value)
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(calls[0]["args"], {"mode": value})
|
||||
self.assertIsInstance(calls[0]["args"]["mode"], str)
|
||||
|
||||
def test_plain_string_none_not_coerced_streaming(self):
|
||||
for value in ("none", "nil", "null"):
|
||||
with self.subTest(value=value):
|
||||
segments = self._configure_segments("mode", value)
|
||||
calls = _stream_segments(segments, self.tools)
|
||||
self.assertEqual(calls[0]["args"], {"mode": value})
|
||||
self.assertIsInstance(calls[0]["args"]["mode"], str)
|
||||
|
||||
def test_streaming_and_non_streaming_agree_for_string(self):
|
||||
for value in ("none", "nil", "null"):
|
||||
with self.subTest(value=value):
|
||||
segments = self._configure_segments("mode", value)
|
||||
non_stream, _ = _parse_segments(segments, self.tools)
|
||||
stream = _stream_segments(segments, self.tools)
|
||||
self.assertEqual(non_stream, stream)
|
||||
|
||||
def test_nullable_param_null_becomes_none(self):
|
||||
segments = self._configure_segments("optional", "null")
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(calls[0]["args"], {"optional": None})
|
||||
|
||||
def test_nullable_param_none_stays_string(self):
|
||||
segments = self._configure_segments("optional", "none")
|
||||
calls, _ = _parse_segments(segments, self.tools)
|
||||
self.assertEqual(calls[0]["args"], {"optional": "none"})
|
||||
|
||||
|
||||
class TestMinimaxM3Streaming(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.tools = _make_tools()
|
||||
|
||||
def test_normal_text_only(self):
|
||||
detector = MinimaxM3Detector()
|
||||
result = detector.parse_streaming_increment("Hello there.", self.tools)
|
||||
self.assertEqual(result.normal_text, "Hello there.")
|
||||
self.assertEqual(len(result.calls), 0)
|
||||
|
||||
def test_single_tool_call_chunked(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
calls = _stream_segments(segments, self.tools)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["name"], "get_weather")
|
||||
self.assertEqual(calls[0]["args"], {"city": "Beijing"})
|
||||
|
||||
def test_streaming_matches_non_streaming(self):
|
||||
cases = {
|
||||
"weather": (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
),
|
||||
"typed": (
|
||||
"<tool_call>",
|
||||
'<invoke name="search">',
|
||||
"<query>pizza",
|
||||
"</query>",
|
||||
"<count>7",
|
||||
"</count>",
|
||||
"<ratio>0.5",
|
||||
"</ratio>",
|
||||
"<verbose>true",
|
||||
"</verbose>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
),
|
||||
"nested": (
|
||||
"<tool_call>",
|
||||
'<invoke name="create_event">',
|
||||
"<title>Standup",
|
||||
"</title>",
|
||||
"<location>",
|
||||
"<city>NYC",
|
||||
"</city>",
|
||||
"<zip>10001",
|
||||
"</zip>",
|
||||
"</location>",
|
||||
"<tags>",
|
||||
"<item>red",
|
||||
"</item>",
|
||||
"<item>blue",
|
||||
"</item>",
|
||||
"</tags>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
),
|
||||
"special": (
|
||||
"<tool_call>",
|
||||
'<invoke name="add_note">',
|
||||
"<note>" + 'a "q" and \\ back\nslash',
|
||||
"</note>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
),
|
||||
"multi_invoke": (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Tokyo",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
),
|
||||
}
|
||||
for name, segments in cases.items():
|
||||
with self.subTest(case=name):
|
||||
non_stream, _ = _parse_segments(segments, self.tools)
|
||||
stream = _stream_segments(segments, self.tools)
|
||||
self.assertEqual(non_stream, stream)
|
||||
|
||||
def test_streaming_sequential_tool_index(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Tokyo",
|
||||
"</city>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
detector = MinimaxM3Detector()
|
||||
all_calls = []
|
||||
for seg in _segments(*segments):
|
||||
all_calls.extend(detector.parse_streaming_increment(seg, self.tools).calls)
|
||||
self.assertEqual(sorted({c.tool_index for c in all_calls}), [0, 1])
|
||||
|
||||
|
||||
class TestMinimaxM3Malformed(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.tools = _make_tools()
|
||||
|
||||
def test_truncated_no_closing_tags(self):
|
||||
text = _wire(
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
)
|
||||
detector = MinimaxM3Detector()
|
||||
result = detector.detect_and_parse(text, self.tools)
|
||||
self.assertEqual(len(result.calls), 0)
|
||||
self.assertEqual(result.normal_text, text)
|
||||
|
||||
def test_mismatched_closing_tag(self):
|
||||
text = _wire(
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
"</wrong>",
|
||||
"</invoke>",
|
||||
"</tool_call>",
|
||||
)
|
||||
detector = MinimaxM3Detector()
|
||||
result = detector.detect_and_parse(text, self.tools)
|
||||
self.assertEqual(len(result.calls), 0)
|
||||
self.assertEqual(result.normal_text, text)
|
||||
|
||||
def test_truncated_streaming_does_not_crash(self):
|
||||
segments = (
|
||||
"<tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
"<city>Beijing",
|
||||
)
|
||||
detector = MinimaxM3Detector()
|
||||
for seg in _segments(*segments):
|
||||
detector.parse_streaming_increment(seg, self.tools)
|
||||
|
||||
|
||||
def _parse_segments_text(text, tools):
|
||||
detector = MinimaxM3Detector()
|
||||
result = detector.detect_and_parse(text, tools)
|
||||
return [
|
||||
{"name": c.name, "args": json.loads(c.parameters)} for c in result.calls
|
||||
], result.normal_text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -762,6 +762,96 @@ class TestMiniMaxAppendThinkDetector(CustomTestCase):
|
||||
self.assertEqual(result.normal_text, "Second")
|
||||
|
||||
|
||||
class TestMiniMaxM3Detector(CustomTestCase):
|
||||
"""Test cases for MiniMaxM3Detector multi-turn stray-closer handling."""
|
||||
|
||||
def _detector(self, force_reasoning=False):
|
||||
from sglang.srt.parser.reasoning_parser import MiniMaxM3Detector
|
||||
|
||||
return MiniMaxM3Detector(force_reasoning=force_reasoning)
|
||||
|
||||
def test_drops_leading_stray_close_non_stream(self):
|
||||
"""Non-thinking multi-turn reply opening with a stray </mm:think>."""
|
||||
result = self._detector().detect_and_parse("</mm:think>The answer is 42.")
|
||||
self.assertEqual(result.normal_text, "The answer is 42.")
|
||||
self.assertEqual(result.reasoning_text or "", "")
|
||||
|
||||
def test_drops_leading_stray_close_with_whitespace(self):
|
||||
result = self._detector().detect_and_parse("\n</mm:think>Hello")
|
||||
self.assertEqual(result.normal_text, "Hello")
|
||||
|
||||
def test_plain_reply_untouched(self):
|
||||
result = self._detector().detect_and_parse("Just a normal answer.")
|
||||
self.assertEqual(result.normal_text, "Just a normal answer.")
|
||||
|
||||
def test_real_reasoning_block_non_stream(self):
|
||||
result = self._detector().detect_and_parse(
|
||||
"<mm:think>reasoning here</mm:think>final"
|
||||
)
|
||||
self.assertEqual(result.reasoning_text, "reasoning here")
|
||||
self.assertEqual(result.normal_text, "final")
|
||||
|
||||
def test_thinking_mode_close_not_dropped(self):
|
||||
"""force_reasoning=True means the closer ends reasoning, not a stray drop."""
|
||||
result = self._detector(force_reasoning=True).detect_and_parse(
|
||||
"reasoning</mm:think>answer"
|
||||
)
|
||||
self.assertEqual(result.reasoning_text, "reasoning")
|
||||
self.assertEqual(result.normal_text, "answer")
|
||||
|
||||
def test_drops_leading_stray_close_stream_single_token(self):
|
||||
detector = self._detector()
|
||||
first = detector.parse_streaming_increment("</mm:think>")
|
||||
self.assertEqual(first.normal_text or "", "")
|
||||
second = detector.parse_streaming_increment("The answer.")
|
||||
self.assertEqual(second.normal_text, "The answer.")
|
||||
|
||||
def test_drops_leading_stray_close_stream_after_whitespace(self):
|
||||
"""A leading whitespace token before the atomic </mm:think> is buffered."""
|
||||
detector = self._detector()
|
||||
self.assertEqual(detector.parse_streaming_increment(" ").normal_text or "", "")
|
||||
self.assertEqual(
|
||||
detector.parse_streaming_increment("</mm:think>").normal_text or "", ""
|
||||
)
|
||||
self.assertEqual(
|
||||
detector.parse_streaming_increment("Hello").normal_text, "Hello"
|
||||
)
|
||||
|
||||
def test_plain_reply_stream_untouched(self):
|
||||
detector = self._detector()
|
||||
out = detector.parse_streaming_increment("Hello")
|
||||
self.assertEqual(out.normal_text, "Hello")
|
||||
|
||||
def test_minimax_m3_model_type(self):
|
||||
from sglang.srt.parser.reasoning_parser import MiniMaxM3Detector
|
||||
|
||||
parser = ReasoningParser("minimax-m3")
|
||||
self.assertIsInstance(parser.detector, MiniMaxM3Detector)
|
||||
|
||||
def test_force_nonempty_content_via_chat_template_kwargs(self):
|
||||
"""force_nonempty_content must reach the M3 detector without a TypeError."""
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionMessageUserParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test",
|
||||
messages=[ChatCompletionMessageUserParam(role="user", content="Hi")],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
parser = ReasoningParser("minimax-m3", request=request)
|
||||
self.assertTrue(parser.detector._force_nonempty_content)
|
||||
|
||||
def test_force_nonempty_content_swaps_when_no_content(self):
|
||||
from sglang.srt.parser.reasoning_parser import MiniMaxM3Detector
|
||||
|
||||
detector = MiniMaxM3Detector(force_reasoning=True, force_nonempty_content=True)
|
||||
result = detector.detect_and_parse("only reasoning, no closer")
|
||||
self.assertEqual(result.normal_text, "only reasoning, no closer")
|
||||
self.assertEqual(result.reasoning_text or "", "")
|
||||
|
||||
|
||||
class TestReasoningParserAdvanced(CustomTestCase):
|
||||
"""Additional tests for ReasoningParser init edge cases."""
|
||||
|
||||
|
||||
@@ -130,6 +130,27 @@ class TestTemplateManagerReasoningDetection(unittest.TestCase):
|
||||
self.assertIsNone(config)
|
||||
self.assertEqual(parser, "minimax")
|
||||
|
||||
MINIMAX_M3_TEMPLATE = (
|
||||
"{%- set ns_token = ']<]minimax[>[' -%}\n"
|
||||
"{%- set toolcall_begin_token = ns_token ~ '<tool_call>' -%}\n"
|
||||
"<mm:think>\n"
|
||||
)
|
||||
|
||||
def test_minimax_m3_detected_via_mm_think_signature(self):
|
||||
_, config, parser = self._detect(
|
||||
self.MINIMAX_M3_TEMPLATE, ["<mm:think>", "</mm:think>"]
|
||||
)
|
||||
|
||||
self.assertIsNone(config)
|
||||
self.assertEqual(parser, "minimax-m3")
|
||||
|
||||
def test_minimax_m2_not_misclassified_as_m3(self):
|
||||
template = """
|
||||
{%- set toolcall_begin_token = '<minimax:tool_call>' -%}
|
||||
"""
|
||||
_, _, parser = self._detect(template, ["<minimax:tool_call>"])
|
||||
self.assertEqual(parser, "minimax")
|
||||
|
||||
|
||||
class TestTemplateDetectionRuleMatrix(unittest.TestCase):
|
||||
"""Table-driven tests for REASONING_PARSER_RULES and REASONING_MODE_RULES."""
|
||||
@@ -376,6 +397,13 @@ class TestToolCallParserDetection(unittest.TestCase):
|
||||
("gpt_oss", "<|channel|>analysis<|message|>", [], "gpt-oss"),
|
||||
("gemma4", "<|channel>content", [], "gemma4"),
|
||||
("minimax_maps_to_m2", "<minimax:tool_call>", [], "minimax-m2"),
|
||||
(
|
||||
"minimax_m3_ns_token_only",
|
||||
"{%- set ns_token = ']<]minimax[>[' -%}\n"
|
||||
"{%- set toolcall_begin_token = ns_token ~ '<tool_call>' -%}",
|
||||
[],
|
||||
"minimax-m3",
|
||||
),
|
||||
(
|
||||
"deepseekv3",
|
||||
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}",
|
||||
@@ -583,6 +611,24 @@ class TestToolCallParserDetection(unittest.TestCase):
|
||||
self.assertLess(minicpm5_idx, rule_names.index("mimo"))
|
||||
self.assertLess(minicpm5_idx, rule_names.index("qwen"))
|
||||
|
||||
def test_minimax_m3_rule_precedes_m2_in_both_registries(self):
|
||||
for rules in (REASONING_PARSER_RULES, TOOL_CALL_PARSER_RULES):
|
||||
names = [rule.name for rule in rules]
|
||||
self.assertLess(names.index("minimax_m3"), names.index("minimax"))
|
||||
|
||||
def test_minicpm5_not_misclassified_as_qwen(self):
|
||||
template = (
|
||||
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}"
|
||||
'\n<function name="{{ tool.name }}">'
|
||||
'\n<param name="{{ param.name }}">{{ param.value }}</param>'
|
||||
"\n</function>"
|
||||
)
|
||||
force, config = detect_reasoning_pattern(template)
|
||||
result = detect_tool_call_parser(
|
||||
template, _DummyTokenizer(["<function", "<param"]), config, force
|
||||
)
|
||||
self.assertEqual(result, "minicpm5")
|
||||
|
||||
|
||||
class TestResolveAutoParsers(unittest.TestCase):
|
||||
"""Tests for resolve_auto_parsers()."""
|
||||
|
||||
Reference in New Issue
Block a user