diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 3a739cfe96..89122c3da7 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1847,6 +1847,11 @@ SGLang supports various environment variables that can be used to configure its Use an explicitly passed --mem-fraction-static as-is on AITER with context length above 8192, instead of scaling it by 0.85 to reserve non-static attention workspace. Skipping the reserve can OOM long-context serving; set it only when the scaled fraction is too small to hold the model weights. false + + SGLANG_AITER_GFX942_BLOCKSCALE_USE_CK + On gfx942, run dense block-FP8 linears with the AITER CK blockscale GEMM instead of the AITER Triton one. Faster, especially at decode batch sizes, but check accuracy: it lowers GSM8K on Qwen3-32B-FP8 by about 2 points. + false + SGLANG_USE_AITER_MOE_GU_ITLV Select the AITER MoE gate/up tile layout: true interleaves, false uses the separated layout. diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 90959d8cd1..8fcfeab052 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -978,6 +978,8 @@ class Envs: SGLANG_AITER_MLA_VERIFY_BACKEND = EnvStr("asm") # Let aiter plan the KV splits for the asm persistent MLA decode. SGLANG_AITER_MLA_AUTO_KV_SPLITS = EnvBool(False) + # Run gfx942 block-FP8 linears with the AITER CK blockscale GEMM instead of Triton. + SGLANG_AITER_GFX942_BLOCKSCALE_USE_CK = EnvBool(False) # DSV4 Aiter flags SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 3b560c587a..92adc1b281 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -45,6 +45,7 @@ from sglang.srt.utils import ( is_cuda, is_flashinfer_available, is_gfx95_supported, + is_gfx942_supported, is_gfx1250_supported, is_hip, is_musa, @@ -74,6 +75,7 @@ _use_aiter = ( get_bool_env_var("SGLANG_USE_AITER") and _is_hip and not _is_gfx1250_supported ) _use_aiter_gfx95 = _use_aiter and _is_gfx95_supported +_use_aiter_gfx942 = _use_aiter and is_gfx942_supported() # Conservative, not a tuned crossover: ptpc beat block-fp8 up to M=512 on # MiniMax-M3 TP4 gfx950 shapes, but lost past M=64 on narrow ones (K=768). MXFP8_DENSE_PTPC_DECODE_MAX_M = 128 @@ -1476,6 +1478,8 @@ def aiter_w8a8_block_fp8_linear( use_triton = use_aiter_triton_gemm_w8a8_tuned_gfx950(n, k) or ( _ck_safe_m is not None and input_2d.shape[0] > _ck_safe_m ) + elif _use_aiter_gfx942: + use_triton = not envs.SGLANG_AITER_GFX942_BLOCKSCALE_USE_CK.get() else: use_triton = True