diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index 4f9c3bcb1..b9e5aa0c7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -405,7 +405,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 -export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 export TASK_QUEUE_ENABLE=1 @@ -428,6 +427,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 3 4 5 6 \ --moe-a2a-backend ascend_fuseep \ + --fuseep-mode 2 \ --deepep-mode auto \ --quantization modelslim \ --speculative-algorithm EAGLE3 \ @@ -509,7 +509,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 -export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 export TASK_QUEUE_ENABLE=1 @@ -531,6 +530,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 4 8 12 16 20 \ --moe-a2a-backend ascend_fuseep \ + --fuseep-mode 2 \ --deepep-mode auto \ --quantization modelslim \ --speculative-algorithm EAGLE3 \ diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index a0248b71e..9cd1abe8d 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -650,11 +650,6 @@ SGLang supports various environment variables that can be used to configure its Enable MoE padding (sets padding size to 128 if value is 1, often set to 1 in Docker builds) `false` - - SGLANG_CUTLASS_MOE (deprecated) - Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass) - `false` - SGLANG_USE_FUSED_PARALLEL_QKNORM Use the fused parallel QK RMSNorm kernel for MiniMax-M2.x on CUDA when attention TP size > 1 @@ -2127,11 +2122,6 @@ SGLang supports various environment variables that can be used to configure its If a warmup forward batch takes longer than this many seconds, the server crashes to avoid hanging. -1 disables; increase (e.g. to 1800) to accommodate kernel JIT precompile. -1 - - SGLANG_ENABLE_GRPC - Enable the native gRPC server (internal, not yet user-facing). - false - SGLANG_GRPC_PORT Port for the native gRPC server. diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 6a516be2b..9313e4e59 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -2522,20 +2522,6 @@ def _a2a_fusion_adjustments(view: Any) -> dict: return {} -def _cutlass_moe_env_override(view: Any) -> dict: - - if envs.SGLANG_CUTLASS_MOE.get(): - logger.warning( - "SGLANG_CUTLASS_MOE is deprecated, use --moe-runner-backend=cutlass and/or --speculative-moe-runner-backend=cutlass instead" - ) - assert view.quantization in [ - "fp8", - "mxfp8", - ], "cutlass MoE is only supported with fp8/mxfp8 quantization" - return {"moe_runner_backend": "cutlass"} - return {} - - # Every A2A backend that forces expert parallelism to span the TP group. _A2A_EP_SPANNING_BACKENDS = frozenset( { diff --git a/python/sglang/srt/disaggregation/common/staging_buffer.py b/python/sglang/srt/disaggregation/common/staging_buffer.py index cba5dd7a6..9ff1d0ef6 100644 --- a/python/sglang/srt/disaggregation/common/staging_buffer.py +++ b/python/sglang/srt/disaggregation/common/staging_buffer.py @@ -13,7 +13,6 @@ Usage: from __future__ import annotations import logging -import os import threading from typing import List, Optional, Tuple @@ -21,11 +20,13 @@ import torch import triton import triton.language as tl +from sglang.srt.environ import envs + logger = logging.getLogger(__name__) # TODO(yangminl): remove torch fallback implementations once the Triton kernels # have been validated in production across all configurations. -_USE_TRITON_STAGING = not bool(os.environ.get("SGLANG_STAGING_USE_TORCH", "")) +_USE_TRITON_STAGING = not envs.SGLANG_STAGING_USE_TORCH.get() @triton.jit diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 6e0a8a62b..5d0f844b2 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1,11 +1,10 @@ import functools import json import os -import subprocess import warnings -from contextlib import ExitStack, contextmanager +from contextlib import contextmanager from enum import IntEnum -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional @functools.lru_cache(maxsize=1) @@ -442,7 +441,6 @@ class Envs: SGLANG_DEBUG_POISON_POOL = EnvBool(False) SGLANG_DEBUG_REVERT_PR = EnvInt(0) SGLANG_PHASE_CHECKER_DEBUG = EnvBool(False) - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False) SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(True) SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0) SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True) @@ -831,9 +829,9 @@ class Envs: SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBoolWithAlias( False, deprecated_name="SGLANG_NPU_USE_TRITON_KV_CACHE_STORE" ) - # Quantize x to int8 in the dispatch operator - DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) # This argument is deprecated - SGLANG_NPU_FUSED_MOE_MODE = EnvInt(1) + # Quantize x to int8 in the dispatch operator (vendor alias consumed by the + # Ascend DeepEP library; the MTP draft-build scopes override it to False). + DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") @@ -992,7 +990,9 @@ class Envs: # =================================================================== # Expert-parallel dispatch and MoE execution # =================================================================== - SGLANG_DEEPEP_BF16_DISPATCH = EnvBool(False) # This argument is deprecated + # Deprecated in favor of '--deepep-dispatcher-output-dtype bf16' but still + # read by several call sites; do not use in new code. + SGLANG_DEEPEP_BF16_DISPATCH = EnvBool(False) SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32) SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO = EnvBool(False) @@ -1052,7 +1052,6 @@ class Envs: SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None) USE_TRITON_W8A8_FP8_KERNEL = EnvBool(False) SGLANG_MOE_PADDING = EnvBool(False) - SGLANG_CUTLASS_MOE = EnvBool(False) # =================================================================== # Logits and log-probability processing @@ -1281,8 +1280,6 @@ class Envs: SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False) SGLANG_EXPERIMENTAL_ONLINE_C128_MTP = EnvBool(False) SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32") - # Deprecated: DSV4 compressor V2 is always used. - SGLANG_OPT_USE_COMPRESSOR_V2 = EnvBool(True) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_TOPK_TRANSFORM_512_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) @@ -1533,24 +1530,122 @@ envs = Envs() EnvField._allow_set_name = False -def _print_deprecated_env(old_name: str, new_name: Optional[str] = None): - if old_name in os.environ: - if new_name is None: - warnings.warn(f"Environment variable {old_name} has been deprecated.") - else: +class _DeprecatedEnv: + """One deprecated env var: warn if it is set, and optionally forward its + (possibly transformed) value to a replacement env var.""" + + def __init__( + self, + replacement: Optional[str] = None, + transform: Optional[Callable[[str], str]] = None, + note: Optional[str] = None, + ): + self.replacement = replacement + self.transform = transform + self.note = note + + def apply(self, old_name: str): + if old_name not in os.environ: + return + message = f"Environment variable {old_name} is deprecated." + if self.replacement is not None: + message += f" Please use {self.replacement} instead." + if self.note is not None: + message += f" {self.note}" + warnings.warn(message) + if self.replacement is not None: + value = os.environ[old_name] + if self.transform is not None: + value = self.transform(value) + os.environ[self.replacement] = value + + +def _ms_to_s(value: str) -> str: + return str(float(value) / 1000.0) + + +def _invert_bool(value: str) -> str: + return "0" if value.lower() in ("true", "1", "yes", "y") else "1" + + +# The single registry for deprecated environment variables, processed once at +# import by _handle_deprecated_envs(). Add new deprecations here instead of +# ad-hoc warnings. For a rename where the old name must keep working through a +# descriptor, use EnvBoolWithAlias / EnvIntWithAlias instead. +_DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = { + # Renamed: the value is forwarded to the replacement. + "SGLANG_GC_LOG": _DeprecatedEnv(replacement="SGLANG_LOG_GC"), + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": _DeprecatedEnv( + replacement="SGLANG_MOE_NVFP4_DISPATCH" + ), + "SGLANG_ENABLE_THINKING": _DeprecatedEnv(replacement="SGLANG_DEFAULT_THINKING"), + "SGLANG_REASONING_EFFORT": _DeprecatedEnv( + replacement="SGLANG_DSV4_REASONING_EFFORT" + ), + "SGLANG_USE_JIT_ALL_REDUCE": _DeprecatedEnv( + replacement="SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2" + ), + # The legacy DISABLE flags have the opposite polarity of their replacement. + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": _DeprecatedEnv( + replacement="SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK", transform=_invert_bool + ), + # Renamed with a unit change. + "SGLANG_QUEUED_TIMEOUT_MS": _DeprecatedEnv( + replacement="SGLANG_REQ_WAITING_TIMEOUT", + transform=_ms_to_s, + note="Note the unit change: milliseconds -> seconds.", + ), + "SGLANG_FORWARD_TIMEOUT_MS": _DeprecatedEnv( + replacement="SGLANG_REQ_RUNNING_TIMEOUT", + transform=_ms_to_s, + note="Note the unit change: milliseconds -> seconds.", + ), + # Removed without replacement. + "SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2": _DeprecatedEnv(), + # Superseded by the unified JIT per_token_group_quant, the default CUDA path. + "SGLANG_OPT_USE_JIT_PER_TOKEN_GROUP_QUANT": _DeprecatedEnv(), + "SGLANG_MASKED_GEMM_FAST_ACT": _DeprecatedEnv(), + "SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN": _DeprecatedEnv(), + # sconv-family kernels always use the CUDA-JIT ports when supported; no toggle. + "SGLANG_OPT_USE_CUDA_SCONV": _DeprecatedEnv(), + # DSV4 compressor V2 is always used. + "SGLANG_OPT_USE_COMPRESSOR_V2": _DeprecatedEnv(), + # Replaced by CLI flags. + "SGLANG_ENABLE_GRPC": _DeprecatedEnv( + note="Please use '--grpc-port' to enable the native gRPC server." + ), + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": _DeprecatedEnv( + note="Please use '--enable-prefill-delayer' instead." + ), + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": _DeprecatedEnv( + note="Please use '--prefill-delayer-max-delay-passes' instead." + ), + "SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK": _DeprecatedEnv( + note="Please use '--prefill-delayer-token-usage-low-watermark' instead." + ), + "SGLANG_CUTLASS_MOE": _DeprecatedEnv( + note="Please use '--moe-runner-backend=cutlass' and/or " + "'--speculative-moe-runner-backend=cutlass' instead." + ), + "SGLANG_DFLASH_PREFILL_REFILL_TARGET": _DeprecatedEnv( + note="DFlash now auto-enables the min-free-slots delay; unset this env. " + "To override the threshold, use '--min-free-slots-delay'." + ), +} + + +def _handle_deprecated_envs(): + for old_name, deprecation in _DEPRECATED_ENVS.items(): + deprecation.apply(old_name) + + # Rewrite the legacy SGL_ prefix to SGLANG_ (names not covered above). + for key, value in list(os.environ.items()): + if key.startswith("SGL_") and key not in _DEPRECATED_ENVS: + new_key = key.replace("SGL_", "SGLANG_", 1) warnings.warn( - f"Environment variable {old_name} will be deprecated, please use {new_name} instead" + f"Environment variable {key} is deprecated, please use {new_key}" ) - os.environ[new_name] = os.environ[old_name] - - -def _warn_deprecated_env_to_cli_flag(env_name: str, suggestion: str): - """Warn when a deprecated environment variable is used. - - This is for env vars that are deprecated in favor of CLI flags. - """ - if env_name in os.environ: - warnings.warn(f"Environment variable {env_name} is deprecated. {suggestion}") + os.environ[new_key] = value def third_party_cache_defaults() -> Dict[str, str]: @@ -1580,157 +1675,11 @@ def redirect_third_party_caches(): os.environ.setdefault(key, value) -def _convert_SGL_to_SGLANG(): - _print_deprecated_env("SGLANG_GC_LOG", "SGLANG_LOG_GC") - _print_deprecated_env( - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH", "SGLANG_MOE_NVFP4_DISPATCH" - ) - _print_deprecated_env( - "SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK", - ) - _print_deprecated_env("SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2") - # Superseded by the unified JIT per_token_group_quant, the default CUDA path. - _print_deprecated_env("SGLANG_OPT_USE_JIT_PER_TOKEN_GROUP_QUANT") - _print_deprecated_env("SGLANG_MASKED_GEMM_FAST_ACT") - _print_deprecated_env("SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN") - # sconv-family kernels always use the CUDA-JIT ports when supported; no toggle. - _print_deprecated_env("SGLANG_OPT_USE_CUDA_SCONV") - _print_deprecated_env("SGLANG_ENABLE_THINKING", "SGLANG_DEFAULT_THINKING") - _print_deprecated_env("SGLANG_REASONING_EFFORT", "SGLANG_DSV4_REASONING_EFFORT") - _print_deprecated_env( - "SGLANG_USE_JIT_ALL_REDUCE", "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2" - ) - _deprecated_ms_to_s = { - "SGLANG_QUEUED_TIMEOUT_MS": "SGLANG_REQ_WAITING_TIMEOUT", - "SGLANG_FORWARD_TIMEOUT_MS": "SGLANG_REQ_RUNNING_TIMEOUT", - } - for old_name, new_name in _deprecated_ms_to_s.items(): - if old_name in os.environ: - ms_val = os.environ[old_name] - warnings.warn( - f"Environment variable {old_name} (in ms) is deprecated, " - f"please use {new_name} (in seconds) instead" - ) - os.environ[new_name] = str(float(ms_val) / 1000.0) +_handle_deprecated_envs() - for key, value in os.environ.items(): - if key.startswith("SGL_"): - new_key = key.replace("SGL_", "SGLANG_", 1) - warnings.warn( - f"Environment variable {key} is deprecated, please use {new_key}" - ) - os.environ[new_key] = value - - -_convert_SGL_to_SGLANG() -_warn_deprecated_env_to_cli_flag( - "SGLANG_ENABLE_GRPC", - "Please use '--grpc-port' to enable the native gRPC server.", -) -_warn_deprecated_env_to_cli_flag( - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE", - "Please use '--enable-prefill-delayer' instead.", -) -_warn_deprecated_env_to_cli_flag( - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES", - "Please use '--prefill-delayer-max-delay-passes' instead.", -) -_warn_deprecated_env_to_cli_flag( - "SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK", - "Please use '--prefill-delayer-token-usage-low-watermark' instead.", -) -_warn_deprecated_env_to_cli_flag( - "SGLANG_DFLASH_PREFILL_REFILL_TARGET", - "DFlash now auto-enables the min-free-slots delay; unset this env. To " - "override the threshold, use '--min-free-slots-delay'.", -) - -# Import cuda_coredump to trigger auto-injection of CUDA env vars -# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees, -# set CUDA_* env vars in the shell before launching Python. -import sglang.srt.debug_utils.cuda_coredump # noqa: F401, E402 # isort: skip - - -def example_with_exit_stack(): - # Use this style of context manager in unit test - exit_stack = ExitStack() - exit_stack.enter_context(envs.SGLANG_TEST_RETRACT.override(False)) - assert envs.SGLANG_TEST_RETRACT.get() is False - exit_stack.close() - assert envs.SGLANG_TEST_RETRACT.get() is None - - -def example_with_subprocess(): - command = ["python", "-c", "import os; print(os.getenv('SGLANG_TEST_RETRACT'))"] - with envs.SGLANG_TEST_RETRACT.override(True): - process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - process.wait() - output = process.stdout.read().decode("utf-8").strip() - assert output == "True" - - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = process.stdout.read().decode("utf-8").strip() - assert output == "None" - - -def example_with_implicit_bool_avoidance(): - @contextmanager - def assert_throws(message_matcher: str): - try: - yield - except Exception as e: - assert message_matcher in str(e), f"{e=}" - print(f"assert_throws find expected error: {e}") - return - raise AssertionError("assert_throws do not see exceptions") - - with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): - if envs.SGLANG_TEST_RETRACT: - pass - - with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): - if (1 != 1) or envs.SGLANG_TEST_RETRACT: - pass - - with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): - if envs.SGLANG_TEST_RETRACT or (1 == 1): - pass - - -def examples(): - # Example usage for envs - envs.SGLANG_TEST_RETRACT.clear() - assert envs.SGLANG_TEST_RETRACT.get() is False - - envs.SGLANG_TEST_RETRACT.set(None) - assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None - - envs.SGLANG_TEST_RETRACT.clear() - assert not envs.SGLANG_TEST_RETRACT.is_set() - - envs.SGLANG_TEST_RETRACT.set(True) - assert envs.SGLANG_TEST_RETRACT.get() is True - - with envs.SGLANG_TEST_RETRACT.override(None): - assert ( - envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None - ) - - assert envs.SGLANG_TEST_RETRACT.get() is True - - envs.SGLANG_TEST_RETRACT.set(None) - with envs.SGLANG_TEST_RETRACT.override(True): - assert envs.SGLANG_TEST_RETRACT.get() is True - - assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None - - example_with_exit_stack() - example_with_subprocess() - example_with_implicit_bool_avoidance() - - -if __name__ == "__main__": - examples() +# Trigger auto-injection of CUDA coredump env vars when SGLANG_CUDA_COREDUMP=1. +# Best-effort; for strict guarantees, set CUDA_* env vars in the shell before +# launching Python. Imported conditionally to keep the default import of this +# module free of non-stdlib side effects. +if envs.SGLANG_CUDA_COREDUMP.get(): + import sglang.srt.debug_utils.cuda_coredump # noqa: F401, E402 # isort: skip diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 8596fd959..078650a37 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -6691,7 +6691,6 @@ class ServerArgs: # (arg_groups/overrides.py: _moe_runner_backend_quant_constraints); # the compatibility asserts and fusion writes stay below. from sglang.srt.arg_groups.overrides import ( - _cutlass_moe_env_override, _moe_runner_backend_quant_constraints, _moe_runner_fusion_disable, run_post_process_pass, @@ -6766,11 +6765,6 @@ class ServerArgs: # invoked here at the legacy write slots. run_post_process_pass(self, _moe_runner_fusion_disable) - # The deprecated SGLANG_CUTLASS_MOE override moved to the pipeline - # (arg_groups/overrides.py: _cutlass_moe_env_override). It sits after - # the fusion blocks above on purpose: they must observe the - # pre-override runner value, exactly as they did imperatively. - run_post_process_pass(self, _cutlass_moe_env_override) if resolved_view(self).moe_runner_backend == "cutlass" and resolved_view( self ).quantization in [ @@ -9695,32 +9689,6 @@ def get_global_server_args() -> ServerArgs: return get_context().server_args -def _has_cli_arg(argv: List[str], flag: str) -> bool: - return any(arg == flag or arg.startswith(f"{flag}=") for arg in argv) - - -def _apply_fuseep_mode_env_compat( - raw_args: argparse.Namespace, argv: List[str] -) -> None: - if not envs.SGLANG_NPU_FUSED_MOE_MODE.is_set() or _has_cli_arg( - argv, "--fuseep-mode" - ): - return - - fuseep_mode = envs.SGLANG_NPU_FUSED_MOE_MODE.get() - if fuseep_mode not in (1, 2): - raise ValueError( - f"Wrong value of SGLANG_NPU_FUSED_MOE_MODE={fuseep_mode}, " - "the NPU only supports 1 or 2." - ) - - logger.warning( - "The env variable SGLANG_NPU_FUSED_MOE_MODE is deprecated and will be " - "removed in a future release. Please use --fuseep-mode instead." - ) - raw_args.fuseep_mode = fuseep_mode - - def prepare_server_args(argv: List[str]) -> ServerArgs: """ Prepare the server arguments from the command line arguments. @@ -9755,8 +9723,6 @@ def prepare_server_args(argv: List[str]) -> ServerArgs: force=True, ) - _apply_fuseep_mode_env_compat(raw_args, argv) - return ServerArgs.from_cli_args(raw_args) diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index 250e3c002..bd7d53bfd 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -26,7 +26,6 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "ASCEND_USE_FIA": "1", "SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_NPU_FUSED_MOE_MODE": "2", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEPEP_HCCL_BUFFSIZE": "1024", @@ -65,6 +64,8 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ 26, "--moe-a2a-backend", "ascend_fuseep", + "--fuseep-mode", + 2, "--deepep-mode", "auto", "--quantization", diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py index 56573a783..0604897bd 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py @@ -31,7 +31,6 @@ class TestAscendDistTimeout(CustomTestCase): os.environ["HCCL_BUFFSIZE"] = "2048" os.environ["SGLANG_ENABLE_OVERLAP_PLAN_STREAM"] = "1" os.environ["SGLANG_ENABLE_SPEC_V2"] = "1" - os.environ["SGLANG_NPU_FUSED_MOE_MODE"] = "1" os.environ["TRANSFORMERS_VERBOSITY"] = "error" cls.env = os.environ.copy() cls.common_args = [ @@ -58,6 +57,8 @@ class TestAscendDistTimeout(CustomTestCase): 2, "--moe-a2a-backend", "ascend_fuseep", + "--fuseep-mode", + 1, "--deepep-mode", "auto", "--speculative-draft-model-quantization", diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py index a948263b8..b5b0f4495 100644 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py @@ -20,7 +20,6 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "ASCEND_USE_FIA": "1", "SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_NPU_FUSED_MOE_MODE": "2", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEPEP_HCCL_BUFFSIZE": "1024", @@ -62,6 +61,8 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ 26, "--moe-a2a-backend", "ascend_fuseep", + "--fuseep-mode", + 2, "--deepep-mode", "auto", "--quantization", diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py index b4dd856cd..1362bf032 100644 --- a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py @@ -26,7 +26,6 @@ QWEN3_235B_ENVS = { "SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING_BS": "27", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416", - "SGLANG_NPU_FUSED_MOE_MODE": "2", } QWEN3_235B_OTHER_ARGS = [ @@ -60,6 +59,8 @@ QWEN3_235B_OTHER_ARGS = [ "--disable-radix-cache", "--moe-a2a-backend", "ascend_fuseep", + "--fuseep-mode", + 2, "--speculative-algorithm", "EAGLE3", "--speculative-draft-model-path", diff --git a/test/registered/unit/test_environ.py b/test/registered/unit/test_environ.py new file mode 100644 index 000000000..42c3db309 --- /dev/null +++ b/test/registered/unit/test_environ.py @@ -0,0 +1,152 @@ +"""Unit tests for sglang.srt.environ: EnvField semantics and the deprecated-env registry.""" + +import os +import re +import subprocess +import sys +import unittest +import warnings +from contextlib import ExitStack + +from sglang.srt.environ import _DEPRECATED_ENVS, _DeprecatedEnv, envs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +class TestEnvField(unittest.TestCase): + def setUp(self): + envs.SGLANG_TEST_RETRACT.clear() + self.addCleanup(envs.SGLANG_TEST_RETRACT.clear) + + def test_set_get_clear_is_set(self): + self.assertFalse(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), False) + + envs.SGLANG_TEST_RETRACT.set(True) + self.assertTrue(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), True) + + envs.SGLANG_TEST_RETRACT.clear() + self.assertFalse(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), False) + + def test_set_to_none_is_distinct_from_clear(self): + envs.SGLANG_TEST_RETRACT.set(None) + self.assertTrue(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIsNone(envs.SGLANG_TEST_RETRACT.get()) + + envs.SGLANG_TEST_RETRACT.clear() + self.assertFalse(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), False) + + def test_override_restores_previous_state(self): + envs.SGLANG_TEST_RETRACT.set(True) + with envs.SGLANG_TEST_RETRACT.override(None): + self.assertTrue(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIsNone(envs.SGLANG_TEST_RETRACT.get()) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), True) + + envs.SGLANG_TEST_RETRACT.set(None) + with envs.SGLANG_TEST_RETRACT.override(True): + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), True) + self.assertTrue(envs.SGLANG_TEST_RETRACT.is_set()) + self.assertIsNone(envs.SGLANG_TEST_RETRACT.get()) + + def test_override_with_exit_stack(self): + envs.SGLANG_TEST_RETRACT.set(None) + exit_stack = ExitStack() + exit_stack.enter_context(envs.SGLANG_TEST_RETRACT.override(False)) + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), False) + exit_stack.close() + self.assertIsNone(envs.SGLANG_TEST_RETRACT.get()) + + def test_override_is_inherited_by_subprocess(self): + command = [ + sys.executable, + "-c", + "import os; print(os.getenv('SGLANG_TEST_RETRACT'))", + ] + with envs.SGLANG_TEST_RETRACT.override(True): + output = subprocess.check_output(command).decode().strip() + self.assertEqual(output, "True") + + output = subprocess.check_output(command).decode().strip() + self.assertEqual(output, "None") + + def test_implicit_bool_raises(self): + message = re.escape( + "Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`" + ) + + with self.assertRaisesRegex(RuntimeError, message): + if envs.SGLANG_TEST_RETRACT: + pass + + with self.assertRaisesRegex(RuntimeError, message): + if (1 != 1) or envs.SGLANG_TEST_RETRACT: + pass + + with self.assertRaisesRegex(RuntimeError, message): + if envs.SGLANG_TEST_RETRACT or (1 == 1): + pass + + def test_invalid_value_warns_and_returns_default(self): + os.environ["SGLANG_TEST_RETRACT"] = "not-a-bool" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.assertIs(envs.SGLANG_TEST_RETRACT.get(), False) + self.assertIn("Invalid value", str(caught[0].message)) + + +class TestDeprecatedEnvRegistry(unittest.TestCase): + def _apply(self, old_name, deprecation): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + deprecation.apply(old_name) + return caught + + def test_removed_env_warns_without_forwarding(self): + old_name = "SGLANG_TEST_REMOVED_ENV" + os.environ[old_name] = "1" + self.addCleanup(os.environ.pop, old_name, None) + + caught = self._apply(old_name, _DeprecatedEnv()) + self.assertIn(f"{old_name} is deprecated", str(caught[0].message)) + + def test_renamed_env_forwards_value(self): + old_name, new_name = "SGLANG_TEST_OLD_ENV", "SGLANG_TEST_NEW_ENV" + os.environ[old_name] = "abc" + self.addCleanup(os.environ.pop, old_name, None) + self.addCleanup(os.environ.pop, new_name, None) + + caught = self._apply(old_name, _DeprecatedEnv(replacement=new_name)) + self.assertIn(new_name, str(caught[0].message)) + self.assertEqual(os.environ[new_name], "abc") + + def test_unset_env_is_a_no_op(self): + caught = self._apply("SGLANG_TEST_UNSET_ENV", _DeprecatedEnv()) + self.assertEqual(len(caught), 0) + + def test_disable_tp_imbalance_check_polarity_is_inverted(self): + old_name = "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK" + new_name = "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK" + os.environ[old_name] = "1" + self.addCleanup(os.environ.pop, old_name, None) + self.addCleanup(os.environ.pop, new_name, None) + + self._apply(old_name, _DEPRECATED_ENVS[old_name]) + self.assertIs(envs.SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK.get(), False) + + def test_ms_to_s_transform(self): + old_name = "SGLANG_QUEUED_TIMEOUT_MS" + os.environ[old_name] = "1500" + self.addCleanup(os.environ.pop, old_name, None) + self.addCleanup(os.environ.pop, "SGLANG_REQ_WAITING_TIMEOUT", None) + + self._apply(old_name, _DEPRECATED_ENVS[old_name]) + self.assertEqual(envs.SGLANG_REQ_WAITING_TIMEOUT.get(), 1.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 73c502b62..1fdecf387 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1773,29 +1773,6 @@ class TestGoldenModelOverrides(_IsolatedPublish): ) self.assertEqual(_moe_runner_backend_quant_constraints(_view()), {}) - def test_cutlass_moe_env_override_pass(self): - from sglang.srt.arg_groups.overrides import ( - ResolvedView, - _cutlass_moe_env_override, - ) - - with patch("sglang.srt.environ.envs.SGLANG_CUTLASS_MOE") as e: - e.get.return_value = True - self.assertEqual( - _cutlass_moe_env_override( - ResolvedView(SimpleNamespace(quantization="fp8")) - ), - {"moe_runner_backend": "cutlass"}, - ) - with self.assertRaises(AssertionError): - _cutlass_moe_env_override( - ResolvedView(SimpleNamespace(quantization=None)) - ) - e.get.return_value = False - self.assertEqual( - _cutlass_moe_env_override(ResolvedView(SimpleNamespace())), {} - ) - def test_gguf_quantization_pass(self): from sglang.srt.arg_groups.overrides import ResolvedView, _gguf_quantization