[Perf] [AMD] dsv4: carry aiter DSv4 GEMM tuned rows and mHC tile_m=16 until aiter ships them (7/8) (#44008)

Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
kk
2026-10-16 02:47:41 -07:00
committed by GitHub
co-authored by Duyi-Wang
parent f05da43d44
commit baf451fcd6
8 changed files with 1116 additions and 0 deletions
+8
View File
@@ -891,6 +891,14 @@ class Envs:
# AMD, ROCm, and AITER
# ===================================================================
SGLANG_USE_AITER = EnvBool(False)
# DeepSeek-V4 in-tree copies of open aiter tuning PRs, each applied only while the
# pinned aiter still ships the pre-PR config (see
# models/deepseek_common/amd/aiter_intree_tune.py). 0 = use aiter as is.
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
SGLANG_AITER_INTREE_GEMM_TUNE = EnvBool(True)
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
SGLANG_AITER_INTREE_MHC_TILE_M16 = EnvBool(True)
# Fuse MiniMax-M3 main/index QK norm + RoPE with main KV and index-K cache
# insertion. Requires an AITER build with the fp8_e4m3_unit cache contract.
# Enabled by default for supported configurations; set to 0 to disable.
@@ -0,0 +1,33 @@
aiter_pr,gfx,cu_num,M,N,K,libtype,kernelId,splitK,us,kernelName,tflops,bw,errRatio,base_libtype,base_kernelId,base_splitK,base_kernelName
6004,gfx950,256,144,2048,7168,triton,0,0,14.98,triton,282.22,1.09,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6004,gfx950,256,160,2048,7168,triton,0,0,14.98,triton,313.60,1.10,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6004,gfx950,256,176,2048,7168,triton,0,0,15.20,triton,340.04,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6004,gfx950,256,192,2048,7168,triton,0,0,15.35,triton,367.17,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6004,gfx950,256,208,2048,7168,triton,0,0,15.93,triton,383.36,1.07,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6004,gfx950,256,224,2048,7168,triton,0,0,16.06,triton,409.43,1.07,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6004,gfx950,256,240,2048,7168,triton,0,0,16.58,triton,424.96,1.05,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6004,gfx950,256,256,2048,7168,triton,0,0,16.67,triton,450.90,1.05,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,1,7168,2048,triton,0,0,6.3242,triton,4.64,2.32,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,2,7168,2048,triton,0,0,6.4960,triton,9.04,2.26,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,4,7168,2048,triton,0,0,6.5740,triton,17.86,2.24,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,48,7168,2048,triton,0,0,7.6509,triton,184.20,2.02,0.0,ck,6,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
6029,gfx950,256,64,7168,2048,triton,0,0,7.7885,triton,241.26,2.02,0.0,ck,6,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
6029,gfx950,256,80,7168,2048,triton,0,0,8.4111,triton,279.25,1.90,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,96,7168,2048,triton,0,0,8.4429,triton,333.84,1.93,0.0,ck,11,2,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,112,7168,2048,triton,0,0,8.4078,triton,391.10,1.96,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,128,7168,2048,triton,0,0,8.4800,triton,443.17,1.98,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,224,7168,2048,asm,16,1,10.9878,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,598.54,1.67,0.0,ck,17,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,416,7168,2048,triton,0,0,15.7676,triton,774.62,1.36,0.0001,ck,14,3,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,448,7168,2048,triton,0,0,15.3484,triton,856.99,1.43,0.0001,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,1,8192,1536,triton,0,0,5.8418,triton,4.31,2.16,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,4,8192,1536,triton,0,0,5.9312,triton,16.97,2.13,0.0,ck,7,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,8,8192,1536,triton,0,0,5.9205,triton,34.01,2.15,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,16,8192,1536,triton,0,0,6.0284,triton,66.79,2.13,0.0,ck,7,2,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6029,gfx950,256,48,8192,1536,triton,0,0,6.6635,triton,181.28,2.02,0.0,ck,6,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
6029,gfx950,256,64,8192,1536,triton,0,0,6.9998,triton,230.09,1.96,0.0,ck,12,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,128,8192,1536,triton,0,0,7.6742,triton,419.75,1.94,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,144,8192,1536,asm,8,1,9.1351,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_48x128E,396.70,1.66,0.0,triton,0,0,triton
6029,gfx950,256,208,8192,1536,asm,16,1,9.6440,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,542.77,1.69,0.0,ck,17,1,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,256,8192,1536,triton,0,0,10.1142,triton,636.97,1.70,0.0,asm,16,1,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E
6029,gfx950,256,416,8192,1536,triton,0,0,13.0308,triton,803.40,1.54,0.0,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
6029,gfx950,256,448,8192,1536,triton,0,0,13.1352,triton,858.33,1.57,0.0,ck,14,2,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
1 aiter_pr gfx cu_num M N K libtype kernelId splitK us kernelName tflops bw errRatio base_libtype base_kernelId base_splitK base_kernelName
2 6004 gfx950 256 144 2048 7168 triton 0 0 14.98 triton 282.22 1.09 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
3 6004 gfx950 256 160 2048 7168 triton 0 0 14.98 triton 313.60 1.10 0.0 ck 12 0 a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
4 6004 gfx950 256 176 2048 7168 triton 0 0 15.20 triton 340.04 1.10 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
5 6004 gfx950 256 192 2048 7168 triton 0 0 15.35 triton 367.17 1.10 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
6 6004 gfx950 256 208 2048 7168 triton 0 0 15.93 triton 383.36 1.07 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
7 6004 gfx950 256 224 2048 7168 triton 0 0 16.06 triton 409.43 1.07 0.0 ck 12 0 a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
8 6004 gfx950 256 240 2048 7168 triton 0 0 16.58 triton 424.96 1.05 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
9 6004 gfx950 256 256 2048 7168 triton 0 0 16.67 triton 450.90 1.05 0.0 ck 12 0 a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
10 6029 gfx950 256 1 7168 2048 triton 0 0 6.3242 triton 4.64 2.32 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
11 6029 gfx950 256 2 7168 2048 triton 0 0 6.4960 triton 9.04 2.26 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
12 6029 gfx950 256 4 7168 2048 triton 0 0 6.5740 triton 17.86 2.24 0.0 ck 7 1 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
13 6029 gfx950 256 48 7168 2048 triton 0 0 7.6509 triton 184.20 2.02 0.0 ck 6 3 a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
14 6029 gfx950 256 64 7168 2048 triton 0 0 7.7885 triton 241.26 2.02 0.0 ck 6 1 a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
15 6029 gfx950 256 80 7168 2048 triton 0 0 8.4111 triton 279.25 1.90 0.0 ck 11 1 a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
16 6029 gfx950 256 96 7168 2048 triton 0 0 8.4429 triton 333.84 1.93 0.0 ck 11 2 a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
17 6029 gfx950 256 112 7168 2048 triton 0 0 8.4078 triton 391.10 1.96 0.0 ck 11 1 a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
18 6029 gfx950 256 128 7168 2048 triton 0 0 8.4800 triton 443.17 1.98 0.0 ck 11 0 a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
19 6029 gfx950 256 224 7168 2048 asm 16 1 10.9878 _ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E 598.54 1.67 0.0 ck 17 0 a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
20 6029 gfx950 256 416 7168 2048 triton 0 0 15.7676 triton 774.62 1.36 0.0001 ck 14 3 a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
21 6029 gfx950 256 448 7168 2048 triton 0 0 15.3484 triton 856.99 1.43 0.0001 ck 14 0 a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
22 6029 gfx950 256 1 8192 1536 triton 0 0 5.8418 triton 4.31 2.16 0.0 ck 7 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
23 6029 gfx950 256 4 8192 1536 triton 0 0 5.9312 triton 16.97 2.13 0.0 ck 7 3 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
24 6029 gfx950 256 8 8192 1536 triton 0 0 5.9205 triton 34.01 2.15 0.0 ck 7 1 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
25 6029 gfx950 256 16 8192 1536 triton 0 0 6.0284 triton 66.79 2.13 0.0 ck 7 2 a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
26 6029 gfx950 256 48 8192 1536 triton 0 0 6.6635 triton 181.28 2.02 0.0 ck 6 0 a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
27 6029 gfx950 256 64 8192 1536 triton 0 0 6.9998 triton 230.09 1.96 0.0 ck 12 1 a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
28 6029 gfx950 256 128 8192 1536 triton 0 0 7.6742 triton 419.75 1.94 0.0 ck 11 0 a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
29 6029 gfx950 256 144 8192 1536 asm 8 1 9.1351 _ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_48x128E 396.70 1.66 0.0 triton 0 0 triton
30 6029 gfx950 256 208 8192 1536 asm 16 1 9.6440 _ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E 542.77 1.69 0.0 ck 17 1 a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
31 6029 gfx950 256 256 8192 1536 triton 0 0 10.1142 triton 636.97 1.70 0.0 asm 16 1 _ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E
32 6029 gfx950 256 416 8192 1536 triton 0 0 13.0308 triton 803.40 1.54 0.0 ck 14 0 a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
33 6029 gfx950 256 448 8192 1536 triton 0 0 13.1352 triton 858.33 1.57 0.0 ck 14 2 a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
@@ -0,0 +1,122 @@
{
"M_LEQ_1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 14
},
"M_LEQ_4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 1,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 8
},
"M_LEQ_8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 1,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 8
},
"M_LEQ_16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 1,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 8
},
"M_LEQ_32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 8
},
"M_LEQ_64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 8
},
"M_LEQ_128": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 4
},
"M_LEQ_256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 4
},
"M_LEQ_512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 4
},
"any": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 4,
"num_warps": 1,
"num_stages": 2,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
}
}
@@ -0,0 +1,122 @@
{
"M_LEQ_1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 4
},
"M_LEQ_4": {
"BLOCK_SIZE_M": 4,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 4
},
"M_LEQ_8": {
"BLOCK_SIZE_M": 4,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 1,
"num_stages": 2,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"M_LEQ_16": {
"BLOCK_SIZE_M": 8,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 4,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 6,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"M_LEQ_32": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 4,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"M_LEQ_256": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"M_LEQ_512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"any": {
"BLOCK_SIZE_M": 256,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 4,
"num_warps": 8,
"num_stages": 2,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
}
}
@@ -0,0 +1,122 @@
{
"M_LEQ_1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 2,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 1,
"num_stages": 2,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 4,
"num_stages": 3,
"waves_per_eu": 1,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
},
"M_LEQ_256": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"M_LEQ_512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": null,
"NUM_KSPLIT": 1
},
"any": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 1,
"num_stages": 2,
"waves_per_eu": 2,
"matrix_instr_nonkdim": 16,
"cache_modifier": ".cg",
"NUM_KSPLIT": 1
}
}
@@ -0,0 +1,364 @@
"""In-tree copies of DeepSeek-V4 tuning changes that are not yet in the aiter pin.
Three open aiter PRs only change *which* existing kernel aiter picks for a shape:
* ROCm/aiter#6004 -- wqkv_a fp8 blockscale GEMM (N=2048, K=7168): cold-weight rows in
``model_configs/dsv4_a8w8_blockscale_bpreshuffle_tuned_gemm.csv`` and the Triton
``GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json``.
* ROCm/aiter#6029 -- wq_b / wo_b fp8 blockscale GEMMs (N=8192 K=1536, N=7168 K=2048):
cold-weight rows in ``model_configs/a8w8_blockscale_bpreshuffle_tuned_gemm_dsv3.csv``
and the two matching Triton JSONs.
* ROCm/aiter#6030 -- mHC fused post+pre on gfx950: tile_m=16 for the FP32 fn decode
GEMM (``aiter.ops.mhc.get_mhc_fused_post_pre_config``, Python only, bit-identical).
``install_aiter_intree_tune()`` registers them at DeepSeek-V4 model construction,
before any GEMM / mHC config lookup:
* CSV rows: aiter's own resolution of the a8w8 blockscale bpreshuffle table (default
CSV + ``model_configs/`` merge, or the user's ``AITER_CONFIG_*`` list) is read, the
in-tree rows *replace* rows with the same (gfx, cu_num, M, N, K) key, and the result
is handed back to aiter as a single file through
``AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE``. A single path skips aiter's merge,
and rows are replaced in place, never appended, so the in-tree rows can never create
the duplicate keys aiter refuses to load.
* Triton JSON: ``gemm_config_utils.load_config_json`` answers the three target paths
with the in-tree JSON (aiter has no search-path override for Triton configs).
* mHC: ``aiter.ops.mhc.get_mhc_fused_post_pre_config`` is wrapped to force tile_m=16.
Upstream-first: every item applies only while aiter still ships the exact pre-PR
content (CSV row = the recorded ``base_*`` kernel; JSON = the recorded sha256; mHC =
the old policy still picks tile_m=32 at decode M). When aiter has merged the PR, or
re-tuned the shape some other way, the item is skipped and aiter's version is used, so
upgrading the aiter pin switches over automatically and never fails at startup.
Switches (default on): ``SGLANG_AITER_INTREE_GEMM_TUNE`` (#6004 + #6029, CSV and
Triton JSON) and ``SGLANG_AITER_INTREE_MHC_TILE_M16`` (#6030). Set to 0 to use the
pinned aiter as is. ``AITER_LOG_TUNED_CONFIG=1`` makes aiter log the row it picks.
"""
from __future__ import annotations
import functools
import hashlib
import json
import logging
import os
import tempfile
from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional
logger = logging.getLogger(__name__)
_CONFIG_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "aiter_intree_configs"
)
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
# Rows of both PRs (column aiter_pr), with the kernel each replaces (base_* columns,
# identical in aiter 02575f1c3a and in the PRs' parents).
INTREE_BPRESHUFFLE_CSV = os.path.join(
_CONFIG_DIR, "a8w8_blockscale_bpreshuffle_tuned_gemm.csv"
)
BPRESHUFFLE_ENV = "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE"
_KEY_COLS = ["gfx", "cu_num", "M", "N", "K"]
_KERNEL_COLS = ["libtype", "kernelId", "splitK", "kernelName"]
_INTREE_ONLY_COLS = ["aiter_pr"] + ["base_" + c for c in _KERNEL_COLS]
TRITON_CONFIG_NAME = "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED"
TRITON_ARCH = "gfx950"
_INTREE_TRITON_DIR = os.path.join(
_CONFIG_DIR, "triton", TRITON_ARCH, "gemm_a8w8_blockscale_preshuffled"
)
# file -> (aiter PR, sha256 of the canonical JSON aiter shipped before the PR)
INTREE_TRITON_JSON = {
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json": (
6004,
"df0cfb42d0e7fd18635950819cf49f85e8b1624c62a21466ac9cef34a5f96e83",
),
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json": (
6029,
"b4c00199f522d29d9d157fd3d3d7a5342637db84ebd23a328f56ec052d446516",
),
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json": (
6029,
"5f22d2e9da4d88d1b4fcf7b08e2b5c71dfa0372ed69fe7df0c10f0a89a9cdc70",
),
}
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
MHC_TILE_M16_MAX_M = 1024
# Decode M where the pre-#6030 gfx950 policy picks tile_m=32 at DeepSeek-V4-Pro's
# hidden size; if aiter already answers 16 for all of them, it has the change.
_MHC_PROBE_HIDDEN = 7168
_MHC_PROBE_MS = (80, 144, 192, 320, 384, 640)
APPLIED, UPSTREAM, SUPERSEDED, MISSING = "applied", "upstream", "superseded", "missing"
@dataclass
class IntreeReport:
"""Per-item outcome: applied / upstream (aiter has the PR) / superseded (aiter
changed it differently) / missing (aiter no longer has the shape or file)."""
csv_rows: Dict[str, int] = field(default_factory=dict)
csv_file: Optional[str] = None
triton_json: Dict[str, str] = field(default_factory=dict)
mhc: Optional[str] = None
notes: List[str] = field(default_factory=list)
_report: Optional[IntreeReport] = None
def _env_on(name: str) -> bool:
from sglang.srt.environ import envs
return getattr(envs, name).get()
# ---------------------------------------------------------------- CSV rows
def _norm_kernel(row) -> tuple:
def as_int(v):
try:
return int(float(v))
except (TypeError, ValueError):
return v
return (
str(row["libtype"]),
as_int(row["kernelId"]),
as_int(row["splitK"]),
str(row["kernelName"]),
)
def apply_bpreshuffle_rows(table, intree):
"""Replace rows of ``table`` (aiter's resolved table) by the in-tree rows.
A row is replaced only when aiter's row for the same key still selects the
recorded pre-PR kernel. Returns ``(new_table, {status: count})``; the key set
of ``table`` is never changed.
"""
stats = {APPLIED: 0, UPSTREAM: 0, SUPERSEDED: 0, MISSING: 0}
if any(c not in table.columns for c in _KEY_COLS + _KERNEL_COLS):
stats[MISSING] = len(intree)
return table, stats
table = table.copy()
index = {}
for i, key in enumerate(
zip(*(table[c].astype(str).str.strip() for c in _KEY_COLS))
):
index.setdefault(key, []).append(table.index[i])
shared = [c for c in table.columns if c in intree.columns]
for _, row in intree.iterrows():
key = tuple(str(row[c]).strip() for c in _KEY_COLS)
hits = index.get(key, [])
if len(hits) != 1:
stats[MISSING] += 1
continue
current = _norm_kernel(table.loc[hits[0]])
if current == _norm_kernel(row):
stats[UPSTREAM] += 1
elif current == _norm_kernel({c: row["base_" + c] for c in _KERNEL_COLS}):
for c in shared:
table.at[hits[0], c] = row[c]
stats[APPLIED] += 1
else:
stats[SUPERSEDED] += 1
return table, stats
def _write_atomic(df, out_dir: str, stem: str) -> str:
text = df.to_csv(index=False)
digest = hashlib.sha256(text.encode()).hexdigest()[:16]
os.makedirs(out_dir, exist_ok=True)
path = os.path.join(out_dir, f"{stem}.{digest}.csv")
if not os.path.exists(path):
tmp = f"{path}.{os.getpid()}.tmp"
with open(tmp, "w") as f:
f.write(text)
os.replace(tmp, path)
return path
def install_bpreshuffle_rows(report: IntreeReport, out_dir: Optional[str] = None):
import pandas as pd
from aiter.jit.core import AITER_CONFIGS
out_dir = out_dir or os.path.join(
tempfile.gettempdir(), f"sglang_aiter_intree_{os.getuid()}"
)
base = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE
intree = pd.read_csv(INTREE_BPRESHUFFLE_CSV)
table, stats = apply_bpreshuffle_rows(pd.read_csv(base), intree)
report.csv_rows = stats
if os.path.dirname(os.path.abspath(base)) == os.path.abspath(out_dir):
# env inherited from a parent process that already installed the rows
report.notes.append(f"{BPRESHUFFLE_ENV} already points at in-tree table")
if stats[APPLIED] == 0:
report.csv_file = base
return
path = _write_atomic(table, out_dir, "a8w8_blockscale_bpreshuffle_tuned_gemm")
prev = os.environ.get(BPRESHUFFLE_ENV)
os.environ[BPRESHUFFLE_ENV] = path
# get_config_file is lru-cached; drop a resolution made before the env change.
cache_clear = getattr(type(AITER_CONFIGS).get_config_file, "cache_clear", None)
if cache_clear is not None:
cache_clear()
resolved = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE
if resolved != path:
if prev is None:
os.environ.pop(BPRESHUFFLE_ENV, None)
else:
os.environ[BPRESHUFFLE_ENV] = prev
if cache_clear is not None:
cache_clear()
raise RuntimeError(f"aiter resolved {resolved}, expected {path}")
report.csv_file = path
# ---------------------------------------------------------------- Triton JSON
def _json_digest(obj) -> str:
return hashlib.sha256(
json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
def select_triton_overrides(
aiter_cfg_dir: str, report: IntreeReport
) -> Dict[str, dict]:
"""Map aiter JSON path -> in-tree JSON, for files aiter still ships pre-PR."""
overrides = {}
for name, (pr, base_digest) in INTREE_TRITON_JSON.items():
with open(os.path.join(_INTREE_TRITON_DIR, name)) as f:
ours = json.load(f)
aiter_path = os.path.normpath(os.path.join(aiter_cfg_dir, name))
try:
with open(aiter_path) as f:
theirs = json.load(f)
except FileNotFoundError:
report.triton_json[name] = MISSING
continue
if theirs == ours:
report.triton_json[name] = UPSTREAM
elif _json_digest(theirs) == base_digest:
overrides[aiter_path] = ours
report.triton_json[name] = APPLIED
else:
report.triton_json[name] = SUPERSEDED
return overrides
def install_triton_json(report: IntreeReport):
import aiter.ops.triton.utils.gemm_config_utils as gcu
from aiter.ops.triton.utils.config_utils import resolve_config_dir
cfg_dir = resolve_config_dir(
"gemm", TRITON_CONFIG_NAME, backend="triton", arch=TRITON_ARCH
)
overrides = select_triton_overrides(cfg_dir, report)
if not overrides:
return
orig = gcu.load_config_json
def load_config_json(fpath, required=True):
hit = overrides.get(os.path.normpath(fpath))
return hit if hit is not None else orig(fpath, required)
load_config_json.__wrapped__ = orig
gcu.load_config_json = load_config_json
gcu._get_gemm_config_cached.cache_clear()
# ---------------------------------------------------------------- mHC tile_m
def make_mhc_tile_m16_config(orig: Callable) -> Callable:
"""aiter#6030: FP32 fn (not packed BF16) decode M <= 1024 uses tile_m=16."""
@functools.lru_cache(maxsize=1024)
def get_mhc_fused_post_pre_config(
m: int,
hidden_size: int,
w_preshuffle_bf16: bool = False,
res_preshuffle: bool = False,
):
split_k, tile_m, tile_n, tile_k = orig(
m,
hidden_size,
w_preshuffle_bf16=w_preshuffle_bf16,
res_preshuffle=res_preshuffle,
)
if not w_preshuffle_bf16 and m <= MHC_TILE_M16_MAX_M:
tile_m = 16
return split_k, tile_m, tile_n, tile_k
get_mhc_fused_post_pre_config.__wrapped__ = orig
return get_mhc_fused_post_pre_config
def mhc_needs_tile_m16(orig: Callable, arch: str) -> bool:
if arch != "gfx950":
return False
return any(orig(m, _MHC_PROBE_HIDDEN)[1] != 16 for m in _MHC_PROBE_MS)
def install_mhc_tile_m16(report: IntreeReport):
import aiter.ops.mhc as aiter_mhc
orig = aiter_mhc.get_mhc_fused_post_pre_config
try:
arch = aiter_mhc.get_gfx_runtime()
except Exception: # noqa: BLE001
arch = "unknown"
if arch != "gfx950":
report.mhc = f"not gfx950 ({arch})"
return
if not mhc_needs_tile_m16(orig, arch):
report.mhc = UPSTREAM
return
aiter_mhc.get_mhc_fused_post_pre_config = make_mhc_tile_m16_config(orig)
report.mhc = APPLIED
# ---------------------------------------------------------------- entry
def install_aiter_intree_tune() -> Optional[IntreeReport]:
"""Idempotent; call before the first aiter GEMM / mHC config lookup."""
global _report
if _report is not None:
return _report
try:
import aiter # noqa: F401
except ImportError:
return None
report = IntreeReport()
steps = []
if _env_on("SGLANG_AITER_INTREE_GEMM_TUNE"):
steps += [install_bpreshuffle_rows, install_triton_json]
if _env_on("SGLANG_AITER_INTREE_MHC_TILE_M16"):
steps.append(install_mhc_tile_m16)
for step in steps:
try:
step(report)
except Exception as err: # noqa: BLE001 - keep aiter's stock behavior
report.notes.append(f"{step.__name__}: {err!r}")
logger.warning("aiter in-tree tune: %s skipped: %r", step.__name__, err)
_report = report
logger.info(
"aiter in-tree tune (aiter#6004/#6029/#6030): csv_rows=%s csv=%s "
"triton_json=%s mhc_tile_m16=%s",
report.csv_rows,
report.csv_file,
report.triton_json,
report.mhc,
)
return report
+8
View File
@@ -4283,6 +4283,14 @@ class DeepseekV4ForCausalLM(nn.Module):
set_force_ck_w8a8(True)
set_batched_rope(True)
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
from sglang.srt.models.deepseek_common.amd.aiter_intree_tune import (
install_aiter_intree_tune,
)
install_aiter_intree_tune()
if envs.SGLANG_DSV4_DENSE_MXSCALE.get():
set_dense_mxscale(True)
self.config = config