From b2cb24995d4adefb2628d73cbf9cf84dae896bc1 Mon Sep 17 00:00:00 2001 From: Aurick Qiao Date: Thu, 8 Oct 2026 12:20:57 -0700 Subject: [PATCH] [Debug] Add opt-in NaN detection before sampling (#42690) Co-authored-by: Qiaolin-Yu --- python/sglang/srt/environ.py | 1 + python/sglang/srt/layers/sampler.py | 3 ++- python/sglang/srt/utils/async_probe.py | 19 +++++++++++++------ python/sglang/srt/utils/invariants.py | 2 ++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 9b5f90d14b..297e48885f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -512,6 +512,7 @@ class Envs: # page alignment). Off in prod; tests turn it on to fail-fast on # numerical / index violations instead of getting silent NaN cascades. SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False) + SGLANG_ENABLE_NAN_LOGITS_CHECK = EnvBool(False) # Signal level for value/index validity checks (nan/inf/oob/...); see # invariants.py. OFF (prod default) runs only the free data layer, WARN # adds throttled logging, STRICT (CI default) crashes on violations. diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index c2b62fc68b..1ab59e5d71 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -23,7 +23,7 @@ from sglang.srt.layers.logprob_processor import ( from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.sampling.sampling_params import TOP_K_ALL -from sglang.srt.utils.async_probe import sanitize_nan_logits +from sglang.srt.utils.async_probe import maybe_detect_nan, sanitize_nan_logits from sglang.srt.utils.common import ( get_bool_env_var, is_cuda, @@ -163,6 +163,7 @@ class Sampler(nn.Module): to get the unique seed for each position. """ logits = logits_output.next_token_logits + maybe_detect_nan(logits, "sampler: next_token_logits") _trace_e2e_sampler( "forward_enter", logits_shape=tuple(logits.shape), diff --git a/python/sglang/srt/utils/async_probe.py b/python/sglang/srt/utils/async_probe.py index abe25cd9a0..1fc50a4790 100644 --- a/python/sglang/srt/utils/async_probe.py +++ b/python/sglang/srt/utils/async_probe.py @@ -1,8 +1,9 @@ """Async invariant probes — fire torch._assert_async without CPU sync. -All probes are gated on SGLANG_ENABLE_ASYNC_ASSERT (default off in prod). -When the gate is on, a violation surfaces as an assertion at the next CUDA -sync point instead of as a silent NaN cascade or illegal-address crash. +NaN probes run when SGLANG_ENABLE_NAN_LOGITS_CHECK or SGLANG_ENABLE_ASYNC_ASSERT +is enabled. Other probes require SGLANG_ENABLE_ASYNC_ASSERT. Both flags default +off. A violation surfaces at the next CUDA sync point instead of a silent NaN +cascade or illegal-address crash. """ import logging @@ -55,7 +56,10 @@ _nan_warner = _AsyncNanWarner() def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""): """Non-fatal counterpart of maybe_detect_nan: throttled sync-free warning instead of crashing. Callers sanitize the tensor themselves.""" - if envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + if ( + envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get() + or envs.SGLANG_ENABLE_ASYNC_ASSERT.get() + ): # The hard assert path already covers detection. return if tensor is None: @@ -64,7 +68,7 @@ def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""): def sanitize_nan_logits(logits: torch.Tensor, msg: str = ""): - """Detect NaN (assert in CI, throttled warning in prod), then sanitize in + """Detect NaN (assert when enabled, otherwise a throttled warning), then sanitize in place: NaN logits (e.g. fp16 activation overflow) are undefined behavior in sampling kernels and can come back as out-of-vocab token ids. +-1e30 rather than dtype min/max because callers divide logits by temperature, @@ -90,7 +94,10 @@ def maybe_assert_sum(tensor: torch.Tensor, expected: int, msg: str = "") -> None def maybe_detect_nan(tensor: Optional[torch.Tensor], msg: str = ""): """Async NaN check — no GPU-CPU sync, error surfaces at next sync point.""" - if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + if not ( + envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get() + or envs.SGLANG_ENABLE_ASYNC_ASSERT.get() + ): return # A None tensor means there is nothing to probe, e.g. hidden_states on # capture_hidden_mode=NULL paths (STANDALONE speculative decoding). diff --git a/python/sglang/srt/utils/invariants.py b/python/sglang/srt/utils/invariants.py index f64d6e2610..111a16fa8b 100644 --- a/python/sglang/srt/utils/invariants.py +++ b/python/sglang/srt/utils/invariants.py @@ -236,6 +236,8 @@ def expect( (`inv.recover`) is applied unconditionally; only detection is gated. Returns the recovered value.""" level = resolve_level() + if isinstance(inv.prop, NotNaN) and envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get(): + level = InvariantCheckLevel.STRICT if level >= InvariantCheckLevel.WARN and value is not None and value.numel() > 0: detail = f"{inv.prop.name}: {msg}" if msg else inv.prop.name _signal(inv.prop.ok(value), inv=inv, level=level, msg=detail)