From a4d0ff3def3dcde573d265305fc2fef32ab3ab6d Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sun, 21 Jun 2026 14:31:35 -0700 Subject: [PATCH] [misc] Make NaN-logit sanitization opt-in (default off) (#28829) Co-authored-by: hnyls2002 --- python/sglang/srt/environ.py | 2 +- scripts/ci/amd/amd_ci_exec.sh | 4 ---- test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py | 1 + 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 44fc621c2..331d8eb3a 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -653,7 +653,7 @@ class Envs: SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False) # Sanitize NaN logits before sampling kernels and log a throttled warning # (see sanitize_nan_logits). - SGLANG_SANITIZE_NAN_LOGITS = EnvBool(True) + SGLANG_SANITIZE_NAN_LOGITS = EnvBool(False) # VLM SGLANG_VLM_CACHE_SIZE_MB = EnvInt(100) diff --git a/scripts/ci/amd/amd_ci_exec.sh b/scripts/ci/amd/amd_ci_exec.sh index d1c94a253..1cf3d09d6 100755 --- a/scripts/ci/amd/amd_ci_exec.sh +++ b/scripts/ci/amd/amd_ci_exec.sh @@ -21,10 +21,6 @@ declare -A ENV_MAP=( # the MXFP4 EAGLE-MTP decode path and abort the queue with an HSA hardware # exception. [SGLANG_ENABLE_ASYNC_ASSERT]=0 - # Disabled on AMD: the per-step NaN-logit probe/sanitize kernels (#27883) in - # the sampler hot path lower single-batch decode fwd_occupancy below the - # base-a sanity threshold. - [SGLANG_SANITIZE_NAN_LOGITS]=0 [SGLANG_USE_AITER]=1 ) diff --git a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py index a4411f1a1..e31eece45 100644 --- a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py @@ -146,6 +146,7 @@ class TestDummyWithSBO(CustomTestCase): # exception, crash-time coredump) so NaN is sanitized with a # warning instead of killing the scheduler. "SGLANG_ENABLE_ASYNC_ASSERT": "0", + "SGLANG_SANITIZE_NAN_LOGITS": "1", "SGLANG_CUDA_COREDUMP": "0", # Already injected into os.environ by the test process when # SGLANG_CUDA_COREDUMP=1, so it must be overridden explicitly.