From baf451fcd6f54962c7db7dc30c636cb534ba1f2a Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:47:41 +0800 Subject: [PATCH] [Perf] [AMD] dsv4: carry aiter DSv4 GEMM tuned rows and mHC tile_m=16 until aiter ships them (7/8) (#44008) Co-authored-by: Duyi-Wang --- python/sglang/srt/environ.py | 8 + ...a8w8_blockscale_bpreshuffle_tuned_gemm.csv | 33 ++ ..._BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json | 122 ++++++ ..._BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json | 122 ++++++ ..._BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json | 122 ++++++ .../deepseek_common/amd/aiter_intree_tune.py | 364 ++++++++++++++++++ python/sglang/srt/models/deepseek_v4.py | 8 + test/registered/amd/test_aiter_intree_tune.py | 337 ++++++++++++++++ 8 files changed, 1116 insertions(+) create mode 100644 python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/a8w8_blockscale_bpreshuffle_tuned_gemm.csv create mode 100644 python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json create mode 100644 python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json create mode 100644 python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json create mode 100644 python/sglang/srt/models/deepseek_common/amd/aiter_intree_tune.py create mode 100644 test/registered/amd/test_aiter_intree_tune.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8772906b2e..fb4e6fe610 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -891,6 +891,14 @@ class Envs: # AMD, ROCm, and AITER # =================================================================== SGLANG_USE_AITER = EnvBool(False) + # DeepSeek-V4 in-tree copies of open aiter tuning PRs, each applied only while the + # pinned aiter still ships the pre-PR config (see + # models/deepseek_common/amd/aiter_intree_tune.py). 0 = use aiter as is. + # TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin + # TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin + SGLANG_AITER_INTREE_GEMM_TUNE = EnvBool(True) + # TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin + SGLANG_AITER_INTREE_MHC_TILE_M16 = EnvBool(True) # Fuse MiniMax-M3 main/index QK norm + RoPE with main KV and index-K cache # insertion. Requires an AITER build with the fp8_e4m3_unit cache contract. # Enabled by default for supported configurations; set to 0 to disable. diff --git a/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/a8w8_blockscale_bpreshuffle_tuned_gemm.csv b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/a8w8_blockscale_bpreshuffle_tuned_gemm.csv new file mode 100644 index 0000000000..df3310149e --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/a8w8_blockscale_bpreshuffle_tuned_gemm.csv @@ -0,0 +1,33 @@ +aiter_pr,gfx,cu_num,M,N,K,libtype,kernelId,splitK,us,kernelName,tflops,bw,errRatio,base_libtype,base_kernelId,base_splitK,base_kernelName +6004,gfx950,256,144,2048,7168,triton,0,0,14.98,triton,282.22,1.09,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6004,gfx950,256,160,2048,7168,triton,0,0,14.98,triton,313.60,1.10,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6004,gfx950,256,176,2048,7168,triton,0,0,15.20,triton,340.04,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6004,gfx950,256,192,2048,7168,triton,0,0,15.35,triton,367.17,1.10,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6004,gfx950,256,208,2048,7168,triton,0,0,15.93,triton,383.36,1.07,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6004,gfx950,256,224,2048,7168,triton,0,0,16.06,triton,409.43,1.07,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6004,gfx950,256,240,2048,7168,triton,0,0,16.58,triton,424.96,1.05,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6004,gfx950,256,256,2048,7168,triton,0,0,16.67,triton,450.90,1.05,0.0,ck,12,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,1,7168,2048,triton,0,0,6.3242,triton,4.64,2.32,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,2,7168,2048,triton,0,0,6.4960,triton,9.04,2.26,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,4,7168,2048,triton,0,0,6.5740,triton,17.86,2.24,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,48,7168,2048,triton,0,0,7.6509,triton,184.20,2.02,0.0,ck,6,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1 +6029,gfx950,256,64,7168,2048,triton,0,0,7.7885,triton,241.26,2.02,0.0,ck,6,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1 +6029,gfx950,256,80,7168,2048,triton,0,0,8.4111,triton,279.25,1.90,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,96,7168,2048,triton,0,0,8.4429,triton,333.84,1.93,0.0,ck,11,2,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,112,7168,2048,triton,0,0,8.4078,triton,391.10,1.96,0.0,ck,11,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,128,7168,2048,triton,0,0,8.4800,triton,443.17,1.98,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,224,7168,2048,asm,16,1,10.9878,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,598.54,1.67,0.0,ck,17,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,416,7168,2048,triton,0,0,15.7676,triton,774.62,1.36,0.0001,ck,14,3,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,448,7168,2048,triton,0,0,15.3484,triton,856.99,1.43,0.0001,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,1,8192,1536,triton,0,0,5.8418,triton,4.31,2.16,0.0,ck,7,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,4,8192,1536,triton,0,0,5.9312,triton,16.97,2.13,0.0,ck,7,3,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,8,8192,1536,triton,0,0,5.9205,triton,34.01,2.15,0.0,ck,7,1,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,16,8192,1536,triton,0,0,6.0284,triton,66.79,2.13,0.0,ck,7,2,a8w8_blockscale_bpreshuffle_1x128x128_256x16x64x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_4_1x1_intrawave_v1 +6029,gfx950,256,48,8192,1536,triton,0,0,6.6635,triton,181.28,2.02,0.0,ck,6,0,a8w8_blockscale_bpreshuffle_1x128x128_256x16x128x256_16x16_16x16_16x16x1_16x16x1_1x16x1x16_8_1x2_intrawave_v1 +6029,gfx950,256,64,8192,1536,triton,0,0,6.9998,triton,230.09,1.96,0.0,ck,12,1,a8w8_blockscale_bpreshuffle_1x128x128_256x32x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,128,8192,1536,triton,0,0,7.6742,triton,419.75,1.94,0.0,ck,11,0,a8w8_blockscale_bpreshuffle_1x128x128_256x32x128x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,144,8192,1536,asm,8,1,9.1351,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_48x128E,396.70,1.66,0.0,triton,0,0,triton +6029,gfx950,256,208,8192,1536,asm,16,1,9.6440,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E,542.77,1.69,0.0,ck,17,1,a8w8_blockscale_bpreshuffle_1x128x128_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,256,8192,1536,triton,0,0,10.1142,triton,636.97,1.70,0.0,asm,16,1,_ZN5aiter42fp8gemm_bf16_blockscale_BpreShuffle_64x128E +6029,gfx950,256,416,8192,1536,triton,0,0,13.0308,triton,803.40,1.54,0.0,ck,14,0,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1 +6029,gfx950,256,448,8192,1536,triton,0,0,13.1352,triton,858.33,1.57,0.0,ck,14,2,a8w8_blockscale_bpreshuffle_1x128x128_256x64x128x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8_2x1_intrawave_v1 diff --git a/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json new file mode 100644 index 0000000000..e40e8ba105 --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json @@ -0,0 +1,122 @@ +{ + "M_LEQ_1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 14 + }, + "M_LEQ_4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 8 + }, + "M_LEQ_8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 8 + }, + "M_LEQ_16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 8 + }, + "M_LEQ_32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 8 + }, + "M_LEQ_64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 8 + }, + "M_LEQ_128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 4 + }, + "M_LEQ_256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 4 + }, + "M_LEQ_512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 4 + }, + "any": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + } +} diff --git a/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json new file mode 100644 index 0000000000..dbb432d88e --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json @@ -0,0 +1,122 @@ +{ + "M_LEQ_1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 4 + }, + "M_LEQ_4": { + "BLOCK_SIZE_M": 4, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 4 + }, + "M_LEQ_8": { + "BLOCK_SIZE_M": 4, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "M_LEQ_16": { + "BLOCK_SIZE_M": 8, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 6, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "M_LEQ_32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "M_LEQ_256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "M_LEQ_512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "any": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + } +} diff --git a/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json new file mode 100644 index 0000000000..501dbd0a52 --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_configs/triton/gfx950/gemm_a8w8_blockscale_preshuffled/GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json @@ -0,0 +1,122 @@ +{ + "M_LEQ_1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + }, + "M_LEQ_256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "M_LEQ_512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": null, + "NUM_KSPLIT": 1 + }, + "any": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "cache_modifier": ".cg", + "NUM_KSPLIT": 1 + } +} diff --git a/python/sglang/srt/models/deepseek_common/amd/aiter_intree_tune.py b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_tune.py new file mode 100644 index 0000000000..6447b5e16a --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/aiter_intree_tune.py @@ -0,0 +1,364 @@ +"""In-tree copies of DeepSeek-V4 tuning changes that are not yet in the aiter pin. + +Three open aiter PRs only change *which* existing kernel aiter picks for a shape: + +* ROCm/aiter#6004 -- wqkv_a fp8 blockscale GEMM (N=2048, K=7168): cold-weight rows in + ``model_configs/dsv4_a8w8_blockscale_bpreshuffle_tuned_gemm.csv`` and the Triton + ``GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json``. +* ROCm/aiter#6029 -- wq_b / wo_b fp8 blockscale GEMMs (N=8192 K=1536, N=7168 K=2048): + cold-weight rows in ``model_configs/a8w8_blockscale_bpreshuffle_tuned_gemm_dsv3.csv`` + and the two matching Triton JSONs. +* ROCm/aiter#6030 -- mHC fused post+pre on gfx950: tile_m=16 for the FP32 fn decode + GEMM (``aiter.ops.mhc.get_mhc_fused_post_pre_config``, Python only, bit-identical). + +``install_aiter_intree_tune()`` registers them at DeepSeek-V4 model construction, +before any GEMM / mHC config lookup: + +* CSV rows: aiter's own resolution of the a8w8 blockscale bpreshuffle table (default + CSV + ``model_configs/`` merge, or the user's ``AITER_CONFIG_*`` list) is read, the + in-tree rows *replace* rows with the same (gfx, cu_num, M, N, K) key, and the result + is handed back to aiter as a single file through + ``AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE``. A single path skips aiter's merge, + and rows are replaced in place, never appended, so the in-tree rows can never create + the duplicate keys aiter refuses to load. +* Triton JSON: ``gemm_config_utils.load_config_json`` answers the three target paths + with the in-tree JSON (aiter has no search-path override for Triton configs). +* mHC: ``aiter.ops.mhc.get_mhc_fused_post_pre_config`` is wrapped to force tile_m=16. + +Upstream-first: every item applies only while aiter still ships the exact pre-PR +content (CSV row = the recorded ``base_*`` kernel; JSON = the recorded sha256; mHC = +the old policy still picks tile_m=32 at decode M). When aiter has merged the PR, or +re-tuned the shape some other way, the item is skipped and aiter's version is used, so +upgrading the aiter pin switches over automatically and never fails at startup. + +Switches (default on): ``SGLANG_AITER_INTREE_GEMM_TUNE`` (#6004 + #6029, CSV and +Triton JSON) and ``SGLANG_AITER_INTREE_MHC_TILE_M16`` (#6030). Set to 0 to use the +pinned aiter as is. ``AITER_LOG_TUNED_CONFIG=1`` makes aiter log the row it picks. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import logging +import os +import tempfile +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + +_CONFIG_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "aiter_intree_configs" +) + +# TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin +# TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin +# Rows of both PRs (column aiter_pr), with the kernel each replaces (base_* columns, +# identical in aiter 02575f1c3a and in the PRs' parents). +INTREE_BPRESHUFFLE_CSV = os.path.join( + _CONFIG_DIR, "a8w8_blockscale_bpreshuffle_tuned_gemm.csv" +) +BPRESHUFFLE_ENV = "AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE" +_KEY_COLS = ["gfx", "cu_num", "M", "N", "K"] +_KERNEL_COLS = ["libtype", "kernelId", "splitK", "kernelName"] +_INTREE_ONLY_COLS = ["aiter_pr"] + ["base_" + c for c in _KERNEL_COLS] + +TRITON_CONFIG_NAME = "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED" +TRITON_ARCH = "gfx950" +_INTREE_TRITON_DIR = os.path.join( + _CONFIG_DIR, "triton", TRITON_ARCH, "gemm_a8w8_blockscale_preshuffled" +) +# file -> (aiter PR, sha256 of the canonical JSON aiter shipped before the PR) +INTREE_TRITON_JSON = { + # TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin + "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=2048-K=7168.json": ( + 6004, + "df0cfb42d0e7fd18635950819cf49f85e8b1624c62a21466ac9cef34a5f96e83", + ), + # TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin + "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=7168-K=2048.json": ( + 6029, + "b4c00199f522d29d9d157fd3d3d7a5342637db84ebd23a328f56ec052d446516", + ), + "GEMM-A8W8_BLOCKSCALE_PRESHUFFLED-N=8192-K=1536.json": ( + 6029, + "5f22d2e9da4d88d1b4fcf7b08e2b5c71dfa0372ed69fe7df0c10f0a89a9cdc70", + ), +} + +# TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin +MHC_TILE_M16_MAX_M = 1024 +# Decode M where the pre-#6030 gfx950 policy picks tile_m=32 at DeepSeek-V4-Pro's +# hidden size; if aiter already answers 16 for all of them, it has the change. +_MHC_PROBE_HIDDEN = 7168 +_MHC_PROBE_MS = (80, 144, 192, 320, 384, 640) + +APPLIED, UPSTREAM, SUPERSEDED, MISSING = "applied", "upstream", "superseded", "missing" + + +@dataclass +class IntreeReport: + """Per-item outcome: applied / upstream (aiter has the PR) / superseded (aiter + changed it differently) / missing (aiter no longer has the shape or file).""" + + csv_rows: Dict[str, int] = field(default_factory=dict) + csv_file: Optional[str] = None + triton_json: Dict[str, str] = field(default_factory=dict) + mhc: Optional[str] = None + notes: List[str] = field(default_factory=list) + + +_report: Optional[IntreeReport] = None + + +def _env_on(name: str) -> bool: + from sglang.srt.environ import envs + + return getattr(envs, name).get() + + +# ---------------------------------------------------------------- CSV rows + + +def _norm_kernel(row) -> tuple: + def as_int(v): + try: + return int(float(v)) + except (TypeError, ValueError): + return v + + return ( + str(row["libtype"]), + as_int(row["kernelId"]), + as_int(row["splitK"]), + str(row["kernelName"]), + ) + + +def apply_bpreshuffle_rows(table, intree): + """Replace rows of ``table`` (aiter's resolved table) by the in-tree rows. + + A row is replaced only when aiter's row for the same key still selects the + recorded pre-PR kernel. Returns ``(new_table, {status: count})``; the key set + of ``table`` is never changed. + """ + stats = {APPLIED: 0, UPSTREAM: 0, SUPERSEDED: 0, MISSING: 0} + if any(c not in table.columns for c in _KEY_COLS + _KERNEL_COLS): + stats[MISSING] = len(intree) + return table, stats + table = table.copy() + index = {} + for i, key in enumerate( + zip(*(table[c].astype(str).str.strip() for c in _KEY_COLS)) + ): + index.setdefault(key, []).append(table.index[i]) + shared = [c for c in table.columns if c in intree.columns] + for _, row in intree.iterrows(): + key = tuple(str(row[c]).strip() for c in _KEY_COLS) + hits = index.get(key, []) + if len(hits) != 1: + stats[MISSING] += 1 + continue + current = _norm_kernel(table.loc[hits[0]]) + if current == _norm_kernel(row): + stats[UPSTREAM] += 1 + elif current == _norm_kernel({c: row["base_" + c] for c in _KERNEL_COLS}): + for c in shared: + table.at[hits[0], c] = row[c] + stats[APPLIED] += 1 + else: + stats[SUPERSEDED] += 1 + return table, stats + + +def _write_atomic(df, out_dir: str, stem: str) -> str: + text = df.to_csv(index=False) + digest = hashlib.sha256(text.encode()).hexdigest()[:16] + os.makedirs(out_dir, exist_ok=True) + path = os.path.join(out_dir, f"{stem}.{digest}.csv") + if not os.path.exists(path): + tmp = f"{path}.{os.getpid()}.tmp" + with open(tmp, "w") as f: + f.write(text) + os.replace(tmp, path) + return path + + +def install_bpreshuffle_rows(report: IntreeReport, out_dir: Optional[str] = None): + import pandas as pd + from aiter.jit.core import AITER_CONFIGS + + out_dir = out_dir or os.path.join( + tempfile.gettempdir(), f"sglang_aiter_intree_{os.getuid()}" + ) + base = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE + intree = pd.read_csv(INTREE_BPRESHUFFLE_CSV) + table, stats = apply_bpreshuffle_rows(pd.read_csv(base), intree) + report.csv_rows = stats + if os.path.dirname(os.path.abspath(base)) == os.path.abspath(out_dir): + # env inherited from a parent process that already installed the rows + report.notes.append(f"{BPRESHUFFLE_ENV} already points at in-tree table") + if stats[APPLIED] == 0: + report.csv_file = base + return + path = _write_atomic(table, out_dir, "a8w8_blockscale_bpreshuffle_tuned_gemm") + prev = os.environ.get(BPRESHUFFLE_ENV) + os.environ[BPRESHUFFLE_ENV] = path + # get_config_file is lru-cached; drop a resolution made before the env change. + cache_clear = getattr(type(AITER_CONFIGS).get_config_file, "cache_clear", None) + if cache_clear is not None: + cache_clear() + resolved = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE + if resolved != path: + if prev is None: + os.environ.pop(BPRESHUFFLE_ENV, None) + else: + os.environ[BPRESHUFFLE_ENV] = prev + if cache_clear is not None: + cache_clear() + raise RuntimeError(f"aiter resolved {resolved}, expected {path}") + report.csv_file = path + + +# ---------------------------------------------------------------- Triton JSON + + +def _json_digest(obj) -> str: + return hashlib.sha256( + json.dumps(obj, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def select_triton_overrides( + aiter_cfg_dir: str, report: IntreeReport +) -> Dict[str, dict]: + """Map aiter JSON path -> in-tree JSON, for files aiter still ships pre-PR.""" + overrides = {} + for name, (pr, base_digest) in INTREE_TRITON_JSON.items(): + with open(os.path.join(_INTREE_TRITON_DIR, name)) as f: + ours = json.load(f) + aiter_path = os.path.normpath(os.path.join(aiter_cfg_dir, name)) + try: + with open(aiter_path) as f: + theirs = json.load(f) + except FileNotFoundError: + report.triton_json[name] = MISSING + continue + if theirs == ours: + report.triton_json[name] = UPSTREAM + elif _json_digest(theirs) == base_digest: + overrides[aiter_path] = ours + report.triton_json[name] = APPLIED + else: + report.triton_json[name] = SUPERSEDED + return overrides + + +def install_triton_json(report: IntreeReport): + import aiter.ops.triton.utils.gemm_config_utils as gcu + from aiter.ops.triton.utils.config_utils import resolve_config_dir + + cfg_dir = resolve_config_dir( + "gemm", TRITON_CONFIG_NAME, backend="triton", arch=TRITON_ARCH + ) + overrides = select_triton_overrides(cfg_dir, report) + if not overrides: + return + orig = gcu.load_config_json + + def load_config_json(fpath, required=True): + hit = overrides.get(os.path.normpath(fpath)) + return hit if hit is not None else orig(fpath, required) + + load_config_json.__wrapped__ = orig + gcu.load_config_json = load_config_json + gcu._get_gemm_config_cached.cache_clear() + + +# ---------------------------------------------------------------- mHC tile_m + + +def make_mhc_tile_m16_config(orig: Callable) -> Callable: + """aiter#6030: FP32 fn (not packed BF16) decode M <= 1024 uses tile_m=16.""" + + @functools.lru_cache(maxsize=1024) + def get_mhc_fused_post_pre_config( + m: int, + hidden_size: int, + w_preshuffle_bf16: bool = False, + res_preshuffle: bool = False, + ): + split_k, tile_m, tile_n, tile_k = orig( + m, + hidden_size, + w_preshuffle_bf16=w_preshuffle_bf16, + res_preshuffle=res_preshuffle, + ) + if not w_preshuffle_bf16 and m <= MHC_TILE_M16_MAX_M: + tile_m = 16 + return split_k, tile_m, tile_n, tile_k + + get_mhc_fused_post_pre_config.__wrapped__ = orig + return get_mhc_fused_post_pre_config + + +def mhc_needs_tile_m16(orig: Callable, arch: str) -> bool: + if arch != "gfx950": + return False + return any(orig(m, _MHC_PROBE_HIDDEN)[1] != 16 for m in _MHC_PROBE_MS) + + +def install_mhc_tile_m16(report: IntreeReport): + import aiter.ops.mhc as aiter_mhc + + orig = aiter_mhc.get_mhc_fused_post_pre_config + try: + arch = aiter_mhc.get_gfx_runtime() + except Exception: # noqa: BLE001 + arch = "unknown" + if arch != "gfx950": + report.mhc = f"not gfx950 ({arch})" + return + if not mhc_needs_tile_m16(orig, arch): + report.mhc = UPSTREAM + return + aiter_mhc.get_mhc_fused_post_pre_config = make_mhc_tile_m16_config(orig) + report.mhc = APPLIED + + +# ---------------------------------------------------------------- entry + + +def install_aiter_intree_tune() -> Optional[IntreeReport]: + """Idempotent; call before the first aiter GEMM / mHC config lookup.""" + global _report + if _report is not None: + return _report + try: + import aiter # noqa: F401 + except ImportError: + return None + report = IntreeReport() + steps = [] + if _env_on("SGLANG_AITER_INTREE_GEMM_TUNE"): + steps += [install_bpreshuffle_rows, install_triton_json] + if _env_on("SGLANG_AITER_INTREE_MHC_TILE_M16"): + steps.append(install_mhc_tile_m16) + for step in steps: + try: + step(report) + except Exception as err: # noqa: BLE001 - keep aiter's stock behavior + report.notes.append(f"{step.__name__}: {err!r}") + logger.warning("aiter in-tree tune: %s skipped: %r", step.__name__, err) + _report = report + logger.info( + "aiter in-tree tune (aiter#6004/#6029/#6030): csv_rows=%s csv=%s " + "triton_json=%s mhc_tile_m16=%s", + report.csv_rows, + report.csv_file, + report.triton_json, + report.mhc, + ) + return report diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 4f097ae353..f109134cc8 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4283,6 +4283,14 @@ class DeepseekV4ForCausalLM(nn.Module): set_force_ck_w8a8(True) set_batched_rope(True) + # TODO(aiter#6004): remove once ROCm/aiter#6004 is in the aiter pin + # TODO(aiter#6029): remove once ROCm/aiter#6029 is in the aiter pin + # TODO(aiter#6030): remove once ROCm/aiter#6030 is in the aiter pin + from sglang.srt.models.deepseek_common.amd.aiter_intree_tune import ( + install_aiter_intree_tune, + ) + + install_aiter_intree_tune() if envs.SGLANG_DSV4_DENSE_MXSCALE.get(): set_dense_mxscale(True) self.config = config diff --git a/test/registered/amd/test_aiter_intree_tune.py b/test/registered/amd/test_aiter_intree_tune.py new file mode 100644 index 0000000000..843130c6f3 --- /dev/null +++ b/test/registered/amd/test_aiter_intree_tune.py @@ -0,0 +1,337 @@ +"""In-tree aiter#6004 / #6029 / #6030 (aiter_intree_tune). + +Guards: the in-tree rows / JSON / mHC policy apply only while aiter ships the pre-PR +config; once aiter has the PR (simulated by handing aiter a table that already has the +PR rows) nothing is installed, aiter's merge does not see duplicate keys, and the +effective table is the PR's either way. The mHC tile_m=16 policy is bit-identical. +""" + +import glob +import json +import os +import shutil +import tempfile +import unittest +from unittest import mock + +import pandas as pd + +from sglang.srt.models.deepseek_common.amd import aiter_intree_tune as it +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd") + +try: + import aiter # noqa: F401 + from aiter.jit.core import AITER_CONFIGS, AITER_ROOT_DIR + + _HAS_AITER = True +except Exception: # noqa: BLE001 + _HAS_AITER = False + + +def _intree(): + return pd.read_csv(it.INTREE_BPRESHUFFLE_CSV) + + +def _as_stock(intree): + """The rows as aiter ships them before the PRs.""" + stock = intree.copy() + for c in it._KERNEL_COLS: + stock[c] = stock["base_" + c] + return stock.drop(columns=it._INTREE_ONLY_COLS) + + +def _keys(df): + return list(zip(*(df[c].astype(str) for c in it._KEY_COLS))) + + +def _kernels_by_key(df): + return {k: it._norm_kernel(r) for k, (_, r) in zip(_keys(df), df.iterrows())} + + +class TestBpreshuffleRows(CustomTestCase): + def test_rows_match_prs(self): + df = _intree() + self.assertEqual(len(df), 32) + self.assertEqual(len(set(_keys(df))), 32) + pr6004 = df[df.aiter_pr == 6004] + self.assertEqual(len(pr6004), 8) + self.assertTrue(((pr6004.N == 2048) & (pr6004.K == 7168)).all()) + pr6029 = df[df.aiter_pr == 6029] + self.assertEqual(len(pr6029), 24) + self.assertEqual(set(zip(pr6029.N, pr6029.K)), {(8192, 1536), (7168, 2048)}) + # only kernel choices change, never to a CK instance aiter may not have built + self.assertTrue(df.libtype.isin(["triton", "asm"]).all()) + + def _other(self): + return pd.DataFrame( + [["gfx950", 256, 1, 4096, 7168, "triton", 0, 0, 1.0, "triton", 1, 1, 0.0]], + columns=[c for c in _as_stock(_intree()).columns], + ) + + def test_stock_aiter_rows_replaced_in_place(self): + stock = pd.concat([_as_stock(_intree()), self._other()], ignore_index=True) + out, stats = it.apply_bpreshuffle_rows(stock, _intree()) + self.assertEqual(stats[it.APPLIED], 32) + self.assertEqual(_keys(out), _keys(stock)) # same keys, same order + self.assertFalse(out.duplicated(subset=it._KEY_COLS).any()) + want = _kernels_by_key(_intree()) + got = _kernels_by_key(out) + for k, v in want.items(): + self.assertEqual(got[k], v) + + def test_merged_aiter_is_left_alone(self): + merged = _intree().drop(columns=it._INTREE_ONLY_COLS) + out, stats = it.apply_bpreshuffle_rows(merged, _intree()) + self.assertEqual(stats[it.APPLIED], 0) + self.assertEqual(stats[it.UPSTREAM], 32) + pd.testing.assert_frame_equal(out, merged) + + def test_superseded_or_missing_rows_prefer_aiter(self): + stock = _as_stock(_intree()) + stock.loc[0, "libtype"] = "asm" + stock.loc[0, "kernelName"] = "some_newer_kernel" + stock = stock.drop(index=1).reset_index(drop=True) + out, stats = it.apply_bpreshuffle_rows(stock, _intree()) + self.assertEqual(stats[it.SUPERSEDED], 1) + self.assertEqual(stats[it.MISSING], 1) + self.assertEqual(stats[it.APPLIED], 30) + self.assertEqual(out.loc[0, "kernelName"], "some_newer_kernel") + + +@unittest.skipUnless(_HAS_AITER, "aiter not installed") +class TestBpreshuffleWithAiter(CustomTestCase): + """Runs aiter's real table resolution (default CSV + model_configs merge).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="sglang_intree_test_") + # keep aiter's merged-table output private to this test + self._tempdir = tempfile.tempdir + tempfile.tempdir = self.tmp + self._env = os.environ.pop(it.BPRESHUFFLE_ENV, None) + self._clear() + + def tearDown(self): + os.environ.pop(it.BPRESHUFFLE_ENV, None) + if self._env is not None: + os.environ[it.BPRESHUFFLE_ENV] = self._env + tempfile.tempdir = self._tempdir + self._clear() + shutil.rmtree(self.tmp, ignore_errors=True) + + @staticmethod + def _clear(): + type(AITER_CONFIGS).get_config_file.cache_clear() + + def _effective(self): + path = AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE + df = pd.read_csv(path) + self.assertFalse(df.duplicated(subset=it._KEY_COLS).any()) + return path, _kernels_by_key(df) + + def _assert_pr_rows(self, kernels): + for k, v in _kernels_by_key(_intree()).items(): + self.assertEqual(kernels[k], v, k) + + def test_install_on_pinned_aiter(self): + report = it.IntreeReport() + it.install_bpreshuffle_rows(report, out_dir=os.path.join(self.tmp, "out")) + stats = report.csv_rows + self.assertEqual(stats[it.SUPERSEDED] + stats[it.MISSING], 0, stats) + self.assertEqual(stats[it.APPLIED] + stats[it.UPSTREAM], 32, stats) + path, kernels = self._effective() + self.assertEqual(path, report.csv_file) + if stats[it.APPLIED]: + self.assertEqual(os.environ[it.BPRESHUFFLE_ENV], path) + self._assert_pr_rows(kernels) + # idempotent: a second process inheriting the env changes nothing + again = it.IntreeReport() + it.install_bpreshuffle_rows(again, out_dir=os.path.join(self.tmp, "out")) + self.assertEqual(again.csv_rows[it.APPLIED], 0) + self.assertEqual(again.csv_file, path) + + def test_aiter_with_prs_merged(self): + """Simulate aiter having merged #6004/#6029: its tables carry the PR rows.""" + cfg = os.path.join(AITER_ROOT_DIR, "aiter", "configs") + name = "a8w8_blockscale_bpreshuffle_tuned_gemm" + files = [os.path.join(cfg, f"{name}.csv")] + sorted( + p + for p in glob.glob(os.path.join(cfg, "model_configs", f"*{name}*.csv")) + if "untuned" not in os.path.basename(p) + ) + new = {k: r for k, (_, r) in zip(_keys(_intree()), _intree().iterrows())} + copies, patched = [], 0 + for p in files: + df = pd.read_csv(p) + if "gfx" in df.columns: + for i, k in enumerate(_keys(df)): + if k in new: + for c in it._KERNEL_COLS: + df.at[df.index[i], c] = new[k][c] + patched += 1 + dst = os.path.join(self.tmp, os.path.basename(p)) + df.to_csv(dst, index=False) + copies.append(dst) + self.assertEqual(patched, 32) + os.environ[it.BPRESHUFFLE_ENV] = os.pathsep.join(copies) + self._clear() + + report = it.IntreeReport() + it.install_bpreshuffle_rows(report, out_dir=os.path.join(self.tmp, "out")) + self.assertEqual(report.csv_rows[it.APPLIED], 0) + self.assertEqual(report.csv_rows[it.UPSTREAM], 32) + # nothing installed: aiter keeps resolving its own (merged) list + self.assertEqual(os.environ[it.BPRESHUFFLE_ENV], os.pathsep.join(copies)) + self.assertFalse(os.path.exists(os.path.join(self.tmp, "out"))) + _, kernels = self._effective() + self._assert_pr_rows(kernels) + + +class TestTritonJson(CustomTestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="sglang_intree_json_") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _ours(self, name): + with open(os.path.join(it._INTREE_TRITON_DIR, name)) as f: + return json.load(f) + + def test_select(self): + names = sorted(it.INTREE_TRITON_JSON) + self.assertEqual(len(names), 3) + stock = {"M_LEQ_1": {"BLOCK_SIZE_M": 8}} + digests = { + n: (pr, it._json_digest(stock)) + for n, (pr, _) in it.INTREE_TRITON_JSON.items() + } + contents = { + names[0]: stock, # pre-PR -> in-tree applies + names[1]: self._ours(names[1]), # aiter merged the PR + # names[2] absent: aiter dropped / renamed the file + } + for n, obj in contents.items(): + with open(os.path.join(self.tmp, n), "w") as f: + json.dump(obj, f, indent=1) + report = it.IntreeReport() + with mock.patch.dict(it.INTREE_TRITON_JSON, digests): + overrides = it.select_triton_overrides(self.tmp, report) + self.assertEqual( + report.triton_json, + {names[0]: it.APPLIED, names[1]: it.UPSTREAM, names[2]: it.MISSING}, + ) + self.assertEqual( + overrides, {os.path.join(self.tmp, names[0]): self._ours(names[0])} + ) + # re-tuned upstream differently -> aiter wins + with open(os.path.join(self.tmp, names[0]), "w") as f: + json.dump({"M_LEQ_1": {"BLOCK_SIZE_M": 64}}, f) + report = it.IntreeReport() + with mock.patch.dict(it.INTREE_TRITON_JSON, digests): + self.assertEqual(it.select_triton_overrides(self.tmp, report), {}) + self.assertEqual(report.triton_json[names[0]], it.SUPERSEDED) + + @unittest.skipUnless(_HAS_AITER, "aiter not installed") + def test_lookup_through_aiter(self): + import aiter.ops.triton.utils.gemm_config_utils as gcu + from aiter.ops.triton.utils._triton import arch_info + + try: + arch = arch_info.get_arch() + except Exception: # noqa: BLE001 + arch = None + if arch != it.TRITON_ARCH: + self.skipTest("gfx950 only") + orig = gcu.load_config_json + report = it.IntreeReport() + try: + it.install_triton_json(report) + self.assertEqual(len(report.triton_json), 3) + for name, status in report.triton_json.items(): + self.assertIn(status, (it.APPLIED, it.UPSTREAM), name) + n, k = ( + int(x.split("=")[1]) + for x in name[len(it.TRITON_CONFIG_NAME) + 1 : -5].split("-") + ) + ours = self._ours(name) + for bucket in (b for b in ours if b.startswith("M_LEQ_")): + m = int(bucket[len("M_LEQ_") :]) + got, tuned = gcu.get_gemm_config( + it.TRITON_CONFIG_NAME, m, n, k, backend="triton" + ) + self.assertTrue(tuned) + self.assertEqual(got, ours[bucket], (name, bucket)) + finally: + gcu.load_config_json = orig + gcu._get_gemm_config_cached.cache_clear() + + +def _stock_gfx950_policy(m, hidden_size, w_preshuffle_bf16=False, res_preshuffle=False): + from aiter.ops.mhc import _mhc_fused_config_gfx950_256 + + return _mhc_fused_config_gfx950_256(m, hidden_size, 256) + + +@unittest.skipUnless(_HAS_AITER, "aiter not installed") +class TestMhcTileM16(CustomTestCase): + def test_policy(self): + self.assertTrue(it.mhc_needs_tile_m16(_stock_gfx950_policy, "gfx950")) + self.assertFalse(it.mhc_needs_tile_m16(_stock_gfx950_policy, "gfx942")) + patched = it.make_mhc_tile_m16_config(_stock_gfx950_policy) + self.assertFalse(it.mhc_needs_tile_m16(patched, "gfx950")) # i.e. "upstream" + saw_32 = False + for m in list(range(1, 1025, 7)) + [1024, 1025, 2048, 4096]: + for bf16 in (False, True): + old = _stock_gfx950_policy(m, 7168, bf16) + new = patched(m, 7168, w_preshuffle_bf16=bf16, res_preshuffle=False) + saw_32 |= old[1] == 32 and not bf16 and m <= 1024 + want_m = 16 if (not bf16 and m <= 1024) else old[1] + self.assertEqual(new, (old[0], want_m, old[2], old[3]), (m, bf16)) + self.assertTrue(saw_32) + + @unittest.skipUnless(__import__("torch").cuda.is_available(), "needs a GPU") + def test_bit_identical_on_gpu(self): + import aiter.ops.mhc as aiter_mhc + import torch + + if aiter_mhc.get_gfx_runtime() != "gfx950": + self.skipTest("gfx950 only") + orig = aiter_mhc.get_mhc_fused_post_pre_config + stock = getattr(orig, "__wrapped__", orig) + hc, hidden = 4, 7168 + hc3 = hc * 2 + hc * hc + g = torch.Generator(device="cuda").manual_seed(0) + + def rnd(*shape, dtype=torch.float32, scale=1.0): + return (torch.randn(*shape, generator=g, device="cuda") * scale).to(dtype) + + try: + for m in (80, 144, 192, 384, 640): + if stock(m, hidden)[1] == 16: + continue # this aiter already picks 16 here + args = ( + rnd(m, hidden, dtype=torch.bfloat16), + rnd(m, hc, hidden, dtype=torch.bfloat16), + rnd(m, hc, 1), + rnd(m, hc, hc), + rnd(hc3, hc * hidden), + rnd(3, scale=0.1), + rnd(hc3, scale=0.1), + ) + outs = [] + for fn in (stock, it.make_mhc_tile_m16_config(stock)): + aiter_mhc.get_mhc_fused_post_pre_config = fn + outs.append( + aiter_mhc.mhc_fused_post_pre(*args, hc_post_mult_value=2.0) + ) + for a, b in zip(*outs): + self.assertTrue(torch.equal(a, b), m) + finally: + aiter_mhc.get_mhc_fused_post_pre_config = orig + + +if __name__ == "__main__": + unittest.main()