[Perf] [AMD] dsv4: carry aiter DSv4 GEMM tuned rows and mHC tile_m=16 until aiter ships them (7/8) (#44008)
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
@@ -891,6 +891,14 @@ class Envs:
|
||||
# AMD, ROCm, and AITER
|
||||
# ===================================================================
|
||||
SGLANG_USE_AITER = EnvBool(False)
|
||||
# DeepSeek-V4 in-tree copies of open aiter tuning PRs, each applied only while the
|
||||
# pinned aiter still ships the pre-PR config (see
|
||||
# models/deepseek_common/amd/aiter_intree_tune.py). 0 = use aiter as is.
|
||||
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
|
||||
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
|
||||
SGLANG_AITER_INTREE_GEMM_TUNE = EnvBool(True)
|
||||
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
|
||||
SGLANG_AITER_INTREE_MHC_TILE_M16 = EnvBool(True)
|
||||
# Fuse MiniMax-M3 main/index QK norm + RoPE with main KV and index-K cache
|
||||
# insertion. Requires an AITER build with the fp8_e4m3_unit cache contract.
|
||||
# Enabled by default for supported configurations; set to 0 to disable.
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
aiter_pr,gfx,cu_num,M,N,K,libtype,kernelId,splitK,us,kernelName,tflops,bw,errRatio,base_libtype,base_kernelId,base_splitK,base_kernelName
|
||||
6004,gfx950,256,144,2048,7168,triton,0,0,14.98,triton,282.22,1.09,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6004,gfx950,256,160,2048,7168,triton,0,0,14.98,triton,313.60,1.10,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6004,gfx950,256,176,2048,7168,triton,0,0,15.20,triton,340.04,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6004,gfx950,256,192,2048,7168,triton,0,0,15.35,triton,367.17,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6004,gfx950,256,208,2048,7168,triton,0,0,15.93,triton,383.36,1.07,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6004,gfx950,256,224,2048,7168,triton,0,0,16.06,triton,409.43,1.07,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6004,gfx950,256,240,2048,7168,triton,0,0,16.58,triton,424.96,1.05,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6004,gfx950,256,256,2048,7168,triton,0,0,16.67,triton,450.90,1.05,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,1,7168,2048,triton,0,0,6.3242,triton,4.64,2.32,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,2,7168,2048,triton,0,0,6.4960,triton,9.04,2.26,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,4,7168,2048,triton,0,0,6.5740,triton,17.86,2.24,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,48,7168,2048,triton,0,0,7.6509,triton,184.20,2.02,0.0,ck,6,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
|
||||
6029,gfx950,256,64,7168,2048,triton,0,0,7.7885,triton,241.26,2.02,0.0,ck,6,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
|
||||
6029,gfx950,256,80,7168,2048,triton,0,0,8.4111,triton,279.25,1.90,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,96,7168,2048,triton,0,0,8.4429,triton,333.84,1.93,0.0,ck,11,2,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,112,7168,2048,triton,0,0,8.4078,triton,391.10,1.96,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,128,7168,2048,triton,0,0,8.4800,triton,443.17,1.98,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,224,7168,2048,asm,16,1,10.9878,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,598.54,1.67,0.0,ck,17,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,416,7168,2048,triton,0,0,15.7676,triton,774.62,1.36,0.0001,ck,14,3,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,448,7168,2048,triton,0,0,15.3484,triton,856.99,1.43,0.0001,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,1,8192,1536,triton,0,0,5.8418,triton,4.31,2.16,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,4,8192,1536,triton,0,0,5.9312,triton,16.97,2.13,0.0,ck,7,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,8,8192,1536,triton,0,0,5.9205,triton,34.01,2.15,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,16,8192,1536,triton,0,0,6.0284,triton,66.79,2.13,0.0,ck,7,2,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1
|
||||
6029,gfx950,256,48,8192,1536,triton,0,0,6.6635,triton,181.28,2.02,0.0,ck,6,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1
|
||||
6029,gfx950,256,64,8192,1536,triton,0,0,6.9998,triton,230.09,1.96,0.0,ck,12,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,128,8192,1536,triton,0,0,7.6742,triton,419.75,1.94,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,144,8192,1536,asm,8,1,9.1351,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_48x128E,396.70,1.66,0.0,triton,0,0,triton
|
||||
6029,gfx950,256,208,8192,1536,asm,16,1,9.6440,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,542.77,1.69,0.0,ck,17,1,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,256,8192,1536,triton,0,0,10.1142,triton,636.97,1.70,0.0,asm,16,1,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E
|
||||
6029,gfx950,256,416,8192,1536,triton,0,0,13.0308,triton,803.40,1.54,0.0,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
6029,gfx950,256,448,8192,1536,triton,0,0,13.1352,triton,858.33,1.57,0.0,ck,14,2,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1
|
||||
|
+122
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"M_LEQ_1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 14
|
||||
},
|
||||
"M_LEQ_4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 1,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 8
|
||||
},
|
||||
"M_LEQ_8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 1,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 8
|
||||
},
|
||||
"M_LEQ_16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 1,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 8
|
||||
},
|
||||
"M_LEQ_32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 8
|
||||
},
|
||||
"M_LEQ_64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 8
|
||||
},
|
||||
"M_LEQ_128": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 4
|
||||
},
|
||||
"M_LEQ_256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 4
|
||||
},
|
||||
"M_LEQ_512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 4
|
||||
},
|
||||
"any": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 4,
|
||||
"num_warps": 1,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"M_LEQ_1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 4
|
||||
},
|
||||
"M_LEQ_4": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 4
|
||||
},
|
||||
"M_LEQ_8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 1,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_16": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 4,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 6,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 4,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_256": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"any": {
|
||||
"BLOCK_SIZE_M": 256,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 4,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"M_LEQ_1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 2,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 1,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 1,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"M_LEQ_512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": null,
|
||||
"NUM_KSPLIT": 1
|
||||
},
|
||||
"any": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 1,
|
||||
"num_stages": 2,
|
||||
"waves_per_eu": 2,
|
||||
"matrix_instr_nonkdim": 16,
|
||||
"cache_modifier": ".cg",
|
||||
"NUM_KSPLIT": 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
"""In-tree copies of DeepSeek-V4 tuning changes that are not yet in the aiter pin.
|
||||
|
||||
Three open aiter PRs only change *which* existing kernel aiter picks for a shape:
|
||||
|
||||
* ROCm/aiter#6004 -- wqkv_a fp8 blockscale GEMM (N=2048, K=7168): cold-weight rows in
|
||||
``model_configs/dsv4_a8w8_blockscale_bpreshuffle_tuned_gemm.csv`` and the Triton
|
||||
``GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json``.
|
||||
* ROCm/aiter#6029 -- wq_b / wo_b fp8 blockscale GEMMs (N=8192 K=1536, N=7168 K=2048):
|
||||
cold-weight rows in ``model_configs/a8w8_blockscale_bpreshuffle_tuned_gemm_dsv3.csv``
|
||||
and the two matching Triton JSONs.
|
||||
* ROCm/aiter#6030 -- mHC fused post+pre on gfx950: tile_m=16 for the FP32 fn decode
|
||||
GEMM (``aiter.ops.mhc.get_mhc_fused_post_pre_config``, Python only, bit-identical).
|
||||
|
||||
``install_aiter_intree_tune()`` registers them at DeepSeek-V4 model construction,
|
||||
before any GEMM / mHC config lookup:
|
||||
|
||||
* CSV rows: aiter's own resolution of the a8w8 blockscale bpreshuffle table (default
|
||||
CSV + ``model_configs/`` merge, or the user's ``AITER_CONFIG_*`` list) is read, the
|
||||
in-tree rows *replace* rows with the same (gfx, cu_num, M, N, K) key, and the result
|
||||
is handed back to aiter as a single file through
|
||||
``AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE``. A single path skips aiter's merge,
|
||||
and rows are replaced in place, never appended, so the in-tree rows can never create
|
||||
the duplicate keys aiter refuses to load.
|
||||
* Triton JSON: ``gemm_config_utils.load_config_json`` answers the three target paths
|
||||
with the in-tree JSON (aiter has no search-path override for Triton configs).
|
||||
* mHC: ``aiter.ops.mhc.get_mhc_fused_post_pre_config`` is wrapped to force tile_m=16.
|
||||
|
||||
Upstream-first: every item applies only while aiter still ships the exact pre-PR
|
||||
content (CSV row = the recorded ``base_*`` kernel; JSON = the recorded sha256; mHC =
|
||||
the old policy still picks tile_m=32 at decode M). When aiter has merged the PR, or
|
||||
re-tuned the shape some other way, the item is skipped and aiter's version is used, so
|
||||
upgrading the aiter pin switches over automatically and never fails at startup.
|
||||
|
||||
Switches (default on): ``SGLANG_AITER_INTREE_GEMM_TUNE`` (#6004 + #6029, CSV and
|
||||
Triton JSON) and ``SGLANG_AITER_INTREE_MHC_TILE_M16`` (#6030). Set to 0 to use the
|
||||
pinned aiter as is. ``AITER_LOG_TUNED_CONFIG=1`` makes aiter log the row it picks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CONFIG_DIR = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "aiter_intree_configs"
|
||||
)
|
||||
|
||||
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
|
||||
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
|
||||
# Rows of both PRs (column aiter_pr), with the kernel each replaces (base_* columns,
|
||||
# identical in aiter 02575f1c3a and in the PRs' parents).
|
||||
INTREE_BPRESHUFFLE_CSV = os.path.join(
|
||||
_CONFIG_DIR, "a8w8_blockscale_bpreshuffle_tuned_gemm.csv"
|
||||
)
|
||||
BPRESHUFFLE_ENV = "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE"
|
||||
_KEY_COLS = ["gfx", "cu_num", "M", "N", "K"]
|
||||
_KERNEL_COLS = ["libtype", "kernelId", "splitK", "kernelName"]
|
||||
_INTREE_ONLY_COLS = ["aiter_pr"] + ["base_" + c for c in _KERNEL_COLS]
|
||||
|
||||
TRITON_CONFIG_NAME = "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED"
|
||||
TRITON_ARCH = "gfx950"
|
||||
_INTREE_TRITON_DIR = os.path.join(
|
||||
_CONFIG_DIR, "triton", TRITON_ARCH, "gemm_a8w8_blockscale_preshuffled"
|
||||
)
|
||||
# file -> (aiter PR, sha256 of the canonical JSON aiter shipped before the PR)
|
||||
INTREE_TRITON_JSON = {
|
||||
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
|
||||
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json": (
|
||||
6004,
|
||||
"df0cfb42d0e7fd18635950819cf49f85e8b1624c62a21466ac9cef34a5f96e83",
|
||||
),
|
||||
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
|
||||
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json": (
|
||||
6029,
|
||||
"b4c00199f522d29d9d157fd3d3d7a5342637db84ebd23a328f56ec052d446516",
|
||||
),
|
||||
"GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json": (
|
||||
6029,
|
||||
"5f22d2e9da4d88d1b4fcf7b08e2b5c71dfa0372ed69fe7df0c10f0a89a9cdc70",
|
||||
),
|
||||
}
|
||||
|
||||
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
|
||||
MHC_TILE_M16_MAX_M = 1024
|
||||
# Decode M where the pre-#6030 gfx950 policy picks tile_m=32 at DeepSeek-V4-Pro's
|
||||
# hidden size; if aiter already answers 16 for all of them, it has the change.
|
||||
_MHC_PROBE_HIDDEN = 7168
|
||||
_MHC_PROBE_MS = (80, 144, 192, 320, 384, 640)
|
||||
|
||||
APPLIED, UPSTREAM, SUPERSEDED, MISSING = "applied", "upstream", "superseded", "missing"
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntreeReport:
|
||||
"""Per-item outcome: applied / upstream (aiter has the PR) / superseded (aiter
|
||||
changed it differently) / missing (aiter no longer has the shape or file)."""
|
||||
|
||||
csv_rows: Dict[str, int] = field(default_factory=dict)
|
||||
csv_file: Optional[str] = None
|
||||
triton_json: Dict[str, str] = field(default_factory=dict)
|
||||
mhc: Optional[str] = None
|
||||
notes: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
_report: Optional[IntreeReport] = None
|
||||
|
||||
|
||||
def _env_on(name: str) -> bool:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
return getattr(envs, name).get()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- CSV rows
|
||||
|
||||
|
||||
def _norm_kernel(row) -> tuple:
|
||||
def as_int(v):
|
||||
try:
|
||||
return int(float(v))
|
||||
except (TypeError, ValueError):
|
||||
return v
|
||||
|
||||
return (
|
||||
str(row["libtype"]),
|
||||
as_int(row["kernelId"]),
|
||||
as_int(row["splitK"]),
|
||||
str(row["kernelName"]),
|
||||
)
|
||||
|
||||
|
||||
def apply_bpreshuffle_rows(table, intree):
|
||||
"""Replace rows of ``table`` (aiter's resolved table) by the in-tree rows.
|
||||
|
||||
A row is replaced only when aiter's row for the same key still selects the
|
||||
recorded pre-PR kernel. Returns ``(new_table, {status: count})``; the key set
|
||||
of ``table`` is never changed.
|
||||
"""
|
||||
stats = {APPLIED: 0, UPSTREAM: 0, SUPERSEDED: 0, MISSING: 0}
|
||||
if any(c not in table.columns for c in _KEY_COLS + _KERNEL_COLS):
|
||||
stats[MISSING] = len(intree)
|
||||
return table, stats
|
||||
table = table.copy()
|
||||
index = {}
|
||||
for i, key in enumerate(
|
||||
zip(*(table[c].astype(str).str.strip() for c in _KEY_COLS))
|
||||
):
|
||||
index.setdefault(key, []).append(table.index[i])
|
||||
shared = [c for c in table.columns if c in intree.columns]
|
||||
for _, row in intree.iterrows():
|
||||
key = tuple(str(row[c]).strip() for c in _KEY_COLS)
|
||||
hits = index.get(key, [])
|
||||
if len(hits) != 1:
|
||||
stats[MISSING] += 1
|
||||
continue
|
||||
current = _norm_kernel(table.loc[hits[0]])
|
||||
if current == _norm_kernel(row):
|
||||
stats[UPSTREAM] += 1
|
||||
elif current == _norm_kernel({c: row["base_" + c] for c in _KERNEL_COLS}):
|
||||
for c in shared:
|
||||
table.at[hits[0], c] = row[c]
|
||||
stats[APPLIED] += 1
|
||||
else:
|
||||
stats[SUPERSEDED] += 1
|
||||
return table, stats
|
||||
|
||||
|
||||
def _write_atomic(df, out_dir: str, stem: str) -> str:
|
||||
text = df.to_csv(index=False)
|
||||
digest = hashlib.sha256(text.encode()).hexdigest()[:16]
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
path = os.path.join(out_dir, f"{stem}.{digest}.csv")
|
||||
if not os.path.exists(path):
|
||||
tmp = f"{path}.{os.getpid()}.tmp"
|
||||
with open(tmp, "w") as f:
|
||||
f.write(text)
|
||||
os.replace(tmp, path)
|
||||
return path
|
||||
|
||||
|
||||
def install_bpreshuffle_rows(report: IntreeReport, out_dir: Optional[str] = None):
|
||||
import pandas as pd
|
||||
from aiter.jit.core import AITER_CONFIGS
|
||||
|
||||
out_dir = out_dir or os.path.join(
|
||||
tempfile.gettempdir(), f"sglang_aiter_intree_{os.getuid()}"
|
||||
)
|
||||
base = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE
|
||||
intree = pd.read_csv(INTREE_BPRESHUFFLE_CSV)
|
||||
table, stats = apply_bpreshuffle_rows(pd.read_csv(base), intree)
|
||||
report.csv_rows = stats
|
||||
if os.path.dirname(os.path.abspath(base)) == os.path.abspath(out_dir):
|
||||
# env inherited from a parent process that already installed the rows
|
||||
report.notes.append(f"{BPRESHUFFLE_ENV} already points at in-tree table")
|
||||
if stats[APPLIED] == 0:
|
||||
report.csv_file = base
|
||||
return
|
||||
path = _write_atomic(table, out_dir, "a8w8_blockscale_bpreshuffle_tuned_gemm")
|
||||
prev = os.environ.get(BPRESHUFFLE_ENV)
|
||||
os.environ[BPRESHUFFLE_ENV] = path
|
||||
# get_config_file is lru-cached; drop a resolution made before the env change.
|
||||
cache_clear = getattr(type(AITER_CONFIGS).get_config_file, "cache_clear", None)
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
resolved = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE
|
||||
if resolved != path:
|
||||
if prev is None:
|
||||
os.environ.pop(BPRESHUFFLE_ENV, None)
|
||||
else:
|
||||
os.environ[BPRESHUFFLE_ENV] = prev
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
raise RuntimeError(f"aiter resolved {resolved}, expected {path}")
|
||||
report.csv_file = path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Triton JSON
|
||||
|
||||
|
||||
def _json_digest(obj) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def select_triton_overrides(
|
||||
aiter_cfg_dir: str, report: IntreeReport
|
||||
) -> Dict[str, dict]:
|
||||
"""Map aiter JSON path -> in-tree JSON, for files aiter still ships pre-PR."""
|
||||
overrides = {}
|
||||
for name, (pr, base_digest) in INTREE_TRITON_JSON.items():
|
||||
with open(os.path.join(_INTREE_TRITON_DIR, name)) as f:
|
||||
ours = json.load(f)
|
||||
aiter_path = os.path.normpath(os.path.join(aiter_cfg_dir, name))
|
||||
try:
|
||||
with open(aiter_path) as f:
|
||||
theirs = json.load(f)
|
||||
except FileNotFoundError:
|
||||
report.triton_json[name] = MISSING
|
||||
continue
|
||||
if theirs == ours:
|
||||
report.triton_json[name] = UPSTREAM
|
||||
elif _json_digest(theirs) == base_digest:
|
||||
overrides[aiter_path] = ours
|
||||
report.triton_json[name] = APPLIED
|
||||
else:
|
||||
report.triton_json[name] = SUPERSEDED
|
||||
return overrides
|
||||
|
||||
|
||||
def install_triton_json(report: IntreeReport):
|
||||
import aiter.ops.triton.utils.gemm_config_utils as gcu
|
||||
from aiter.ops.triton.utils.config_utils import resolve_config_dir
|
||||
|
||||
cfg_dir = resolve_config_dir(
|
||||
"gemm", TRITON_CONFIG_NAME, backend="triton", arch=TRITON_ARCH
|
||||
)
|
||||
overrides = select_triton_overrides(cfg_dir, report)
|
||||
if not overrides:
|
||||
return
|
||||
orig = gcu.load_config_json
|
||||
|
||||
def load_config_json(fpath, required=True):
|
||||
hit = overrides.get(os.path.normpath(fpath))
|
||||
return hit if hit is not None else orig(fpath, required)
|
||||
|
||||
load_config_json.__wrapped__ = orig
|
||||
gcu.load_config_json = load_config_json
|
||||
gcu._get_gemm_config_cached.cache_clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- mHC tile_m
|
||||
|
||||
|
||||
def make_mhc_tile_m16_config(orig: Callable) -> Callable:
|
||||
"""aiter#6030: FP32 fn (not packed BF16) decode M <= 1024 uses tile_m=16."""
|
||||
|
||||
@functools.lru_cache(maxsize=1024)
|
||||
def get_mhc_fused_post_pre_config(
|
||||
m: int,
|
||||
hidden_size: int,
|
||||
w_preshuffle_bf16: bool = False,
|
||||
res_preshuffle: bool = False,
|
||||
):
|
||||
split_k, tile_m, tile_n, tile_k = orig(
|
||||
m,
|
||||
hidden_size,
|
||||
w_preshuffle_bf16=w_preshuffle_bf16,
|
||||
res_preshuffle=res_preshuffle,
|
||||
)
|
||||
if not w_preshuffle_bf16 and m <= MHC_TILE_M16_MAX_M:
|
||||
tile_m = 16
|
||||
return split_k, tile_m, tile_n, tile_k
|
||||
|
||||
get_mhc_fused_post_pre_config.__wrapped__ = orig
|
||||
return get_mhc_fused_post_pre_config
|
||||
|
||||
|
||||
def mhc_needs_tile_m16(orig: Callable, arch: str) -> bool:
|
||||
if arch != "gfx950":
|
||||
return False
|
||||
return any(orig(m, _MHC_PROBE_HIDDEN)[1] != 16 for m in _MHC_PROBE_MS)
|
||||
|
||||
|
||||
def install_mhc_tile_m16(report: IntreeReport):
|
||||
import aiter.ops.mhc as aiter_mhc
|
||||
|
||||
orig = aiter_mhc.get_mhc_fused_post_pre_config
|
||||
try:
|
||||
arch = aiter_mhc.get_gfx_runtime()
|
||||
except Exception: # noqa: BLE001
|
||||
arch = "unknown"
|
||||
if arch != "gfx950":
|
||||
report.mhc = f"not gfx950 ({arch})"
|
||||
return
|
||||
if not mhc_needs_tile_m16(orig, arch):
|
||||
report.mhc = UPSTREAM
|
||||
return
|
||||
aiter_mhc.get_mhc_fused_post_pre_config = make_mhc_tile_m16_config(orig)
|
||||
report.mhc = APPLIED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- entry
|
||||
|
||||
|
||||
def install_aiter_intree_tune() -> Optional[IntreeReport]:
|
||||
"""Idempotent; call before the first aiter GEMM / mHC config lookup."""
|
||||
global _report
|
||||
if _report is not None:
|
||||
return _report
|
||||
try:
|
||||
import aiter # noqa: F401
|
||||
except ImportError:
|
||||
return None
|
||||
report = IntreeReport()
|
||||
steps = []
|
||||
if _env_on("SGLANG_AITER_INTREE_GEMM_TUNE"):
|
||||
steps += [install_bpreshuffle_rows, install_triton_json]
|
||||
if _env_on("SGLANG_AITER_INTREE_MHC_TILE_M16"):
|
||||
steps.append(install_mhc_tile_m16)
|
||||
for step in steps:
|
||||
try:
|
||||
step(report)
|
||||
except Exception as err: # noqa: BLE001 - keep aiter's stock behavior
|
||||
report.notes.append(f"{step.__name__}: {err!r}")
|
||||
logger.warning("aiter in-tree tune: %s skipped: %r", step.__name__, err)
|
||||
_report = report
|
||||
logger.info(
|
||||
"aiter in-tree tune (aiter#6004/#6029/#6030): csv_rows=%s csv=%s "
|
||||
"triton_json=%s mhc_tile_m16=%s",
|
||||
report.csv_rows,
|
||||
report.csv_file,
|
||||
report.triton_json,
|
||||
report.mhc,
|
||||
)
|
||||
return report
|
||||
@@ -4283,6 +4283,14 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
set_force_ck_w8a8(True)
|
||||
set_batched_rope(True)
|
||||
# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin
|
||||
# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin
|
||||
# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin
|
||||
from sglang.srt.models.deepseek_common.amd.aiter_intree_tune import (
|
||||
install_aiter_intree_tune,
|
||||
)
|
||||
|
||||
install_aiter_intree_tune()
|
||||
if envs.SGLANG_DSV4_DENSE_MXSCALE.get():
|
||||
set_dense_mxscale(True)
|
||||
self.config = config
|
||||
|
||||
Reference in New Issue
Block a user