diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..9e03729be5 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,32 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + } +} \ No newline at end of file diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..9e03729be5 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,32 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + } +} \ No newline at end of file diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..429c7034da --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,32 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + } +} \ No newline at end of file diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 6e33062600..8c306e2413 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -467,8 +467,13 @@ def _apply_wo_a_bf16_matmul( is_prefill: bool = False, fast_path: bool = False, ) -> torch.Tensor | Mxfp8SwizzledInput: - # o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated - # on the exact validated TP4 shapes and write token-major output directly. + """Compute bf16 wo_a: o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]. + + Single-token decode uses a GEMV for the validated TP4/TP8 shapes. Blackwell + verify batches up to 384 rows write token-major output directly to avoid + the layout copy before wo_b. ROCm decode can use aiter batched GEMM; + other cases use torch.einsum. + """ global _wo_a_aiter_batched_gemm_disabled if ( fast_path @@ -490,12 +495,13 @@ def _apply_wo_a_bf16_matmul( and get_platform().is_blackwell ) ) - and o.shape[1:] == (2, 4096) - and wo_a.shape == (2, 1024, 4096) + and o.shape[2] == 4096 + and o.shape[1] in (1, 2) + and wo_a.shape == (o.shape[1], 1024, 4096) and o.dtype == wo_a.dtype == torch.bfloat16 and o.stride(2) == 1 and o.stride(1) == 4096 - and o.stride(0) >= 8192 + and o.stride(0) >= 4096 * o.shape[1] and wo_a.is_contiguous() ): if is_decode and o.shape[0] == 1: