minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Tiny] Renaming environ for NVFP4 dispatch (#13756)

Browse Source
This commit is contained in:
Baizhou Zhang
2025-11-22 00:05:20 -08:00
committed by GitHub
parent b41afa3754
commit 8bfce9b08d
4 changed files with 8 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/references/environment_variables.md
+1 -1
View File
@@ -49,7 +49,7 @@ SGLang supports various environment variables that can be used to configure its
| Environment Variable | Description | Default Value | | Environment Variable | Description | Default Value |
| --- | --- | --- | | --- | --- | --- |
| `SGLANG_DEEPEP_BF16_DISPATCH` | Use Bfloat16 for dispatch | `"false"` | | `SGLANG_DEEPEP_BF16_DISPATCH` | Use Bfloat16 for dispatch | `"false"` |
| `SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH` | Use nvfp4 for dispatch | `"false"` | | `SGLANG_MOE_NVFP4_DISPATCH` | Use nvfp4 for moe dispatch | `"false"` |
## Memory Management ## Memory Management
python/sglang/srt/environ.py
+1
View File
@@ -215,6 +215,7 @@ class Envs:
SGLANG_CPU_QUANTIZATION = EnvBool(False) SGLANG_CPU_QUANTIZATION = EnvBool(False)
SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False) SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False)
SGLANG_FORCE_FP8_MARLIN = EnvBool(False) SGLANG_FORCE_FP8_MARLIN = EnvBool(False)
SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False)
# Flashinfer # Flashinfer
SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True)
python/sglang/srt/layers/quantization/modelopt_quant.py
+4 -6
View File
@@ -139,9 +139,7 @@ CUTEDSL_MOE_SCALAR_INPUT_SCALE = get_bool_env_var(
) )
# TODO make it true by default when the DeepEP PR is merged # TODO make it true by default when the DeepEP PR is merged
CUTEDSL_MOE_NVFP4_DISPATCH = get_bool_env_var( MOE_NVFP4_DISPATCH = envs.SGLANG_MOE_NVFP4_DISPATCH.get()
"SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH", "false"
)
FLASHINFER_FP4_GEMM_BACKEND = envs.SGLANG_FLASHINFER_FP4_GEMM_BACKEND.get() FLASHINFER_FP4_GEMM_BACKEND = envs.SGLANG_FLASHINFER_FP4_GEMM_BACKEND.get()
# Supported activation schemes for the current configuration # Supported activation schemes for the current configuration
ACTIVATION_SCHEMES = ["static"] ACTIVATION_SCHEMES = ["static"]
@@ -1492,7 +1490,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
w13_input_scale = _slice_scale(w13_input_scale) w13_input_scale = _slice_scale(w13_input_scale)
w2_input_scale = _slice_scale(w2_input_scale) w2_input_scale = _slice_scale(w2_input_scale)
if CUTEDSL_MOE_NVFP4_DISPATCH: if MOE_NVFP4_DISPATCH:
assert torch.all(w13_input_scale == w13_input_scale[0]) assert torch.all(w13_input_scale == w13_input_scale[0])
w13_input_scale = w13_input_scale[0] w13_input_scale = w13_input_scale[0]
else: else:
@@ -1518,7 +1516,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
layer.dispatcher.set_quant_config( layer.dispatcher.set_quant_config(
{ {
"input_global_scale": ( "input_global_scale": (
layer.w13_input_scale_quant if CUTEDSL_MOE_NVFP4_DISPATCH else None layer.w13_input_scale_quant if MOE_NVFP4_DISPATCH else None
) )
} }
) )
@@ -1740,7 +1738,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
out = flashinfer_cutedsl_moe_masked( out = flashinfer_cutedsl_moe_masked(
hidden_states=x, hidden_states=x,
input_global_scale=( input_global_scale=(
None if CUTEDSL_MOE_NVFP4_DISPATCH else layer.w13_input_scale_quant None if MOE_NVFP4_DISPATCH else layer.w13_input_scale_quant
), ),
w1=layer.w13_weight, w1=layer.w13_weight,
w1_blockscale=layer.w13_blockscale_swizzled, w1_blockscale=layer.w13_blockscale_swizzled,
test/srt/test_deepseek_v3_cutedsl_4gpu.py
+2 -2
View File
@@ -60,7 +60,7 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase):
**os.environ, **os.environ,
"SGLANG_DEEPEP_BF16_DISPATCH": "1", "SGLANG_DEEPEP_BF16_DISPATCH": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
"SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "0", "SGLANG_MOE_NVFP4_DISPATCH": "0",
}, },
) )
@@ -136,7 +136,7 @@ class TestDummyWithSBO(CustomTestCase):
**os.environ, **os.environ,
"SGLANG_DEEPEP_BF16_DISPATCH": "1", "SGLANG_DEEPEP_BF16_DISPATCH": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
"SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "0", "SGLANG_MOE_NVFP4_DISPATCH": "0",
}, },
) )