[Debug] Add opt-in NaN detection before sampling (#42690)

Co-authored-by: Qiaolin-Yu <liin1211@outlook.com>
This commit is contained in:
Aurick Qiao
2026-10-08 12:20:57 -07:00
committed by GitHub
co-authored by Qiaolin-Yu
parent 393eaed509
commit b2cb24995d
4 changed files with 18 additions and 7 deletions
+1
View File
@@ -512,6 +512,7 @@ class Envs:
# page alignment). Off in prod; tests turn it on to fail-fast on
# numerical / index violations instead of getting silent NaN cascades.
SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False)
SGLANG_ENABLE_NAN_LOGITS_CHECK = EnvBool(False)
# Signal level for value/index validity checks (nan/inf/oob/...); see
# invariants.py. OFF (prod default) runs only the free data layer, WARN
# adds throttled logging, STRICT (CI default) crashes on violations.
+2 -1
View File
@@ -23,7 +23,7 @@ from sglang.srt.layers.logprob_processor import (
from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.sampling.sampling_params import TOP_K_ALL
from sglang.srt.utils.async_probe import sanitize_nan_logits
from sglang.srt.utils.async_probe import maybe_detect_nan, sanitize_nan_logits
from sglang.srt.utils.common import (
get_bool_env_var,
is_cuda,
@@ -163,6 +163,7 @@ class Sampler(nn.Module):
to get the unique seed for each position.
"""
logits = logits_output.next_token_logits
maybe_detect_nan(logits, "sampler: next_token_logits")
_trace_e2e_sampler(
"forward_enter",
logits_shape=tuple(logits.shape),
+13 -6
View File
@@ -1,8 +1,9 @@
"""Async invariant probes — fire torch._assert_async without CPU sync.
All probes are gated on SGLANG_ENABLE_ASYNC_ASSERT (default off in prod).
When the gate is on, a violation surfaces as an assertion at the next CUDA
sync point instead of as a silent NaN cascade or illegal-address crash.
NaN probes run when SGLANG_ENABLE_NAN_LOGITS_CHECK or SGLANG_ENABLE_ASYNC_ASSERT
is enabled. Other probes require SGLANG_ENABLE_ASYNC_ASSERT. Both flags default
off. A violation surfaces at the next CUDA sync point instead of a silent NaN
cascade or illegal-address crash.
"""
import logging
@@ -55,7 +56,10 @@ _nan_warner = _AsyncNanWarner()
def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""):
"""Non-fatal counterpart of maybe_detect_nan: throttled sync-free warning
instead of crashing. Callers sanitize the tensor themselves."""
if envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
if (
envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get()
or envs.SGLANG_ENABLE_ASYNC_ASSERT.get()
):
# The hard assert path already covers detection.
return
if tensor is None:
@@ -64,7 +68,7 @@ def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""):
def sanitize_nan_logits(logits: torch.Tensor, msg: str = ""):
"""Detect NaN (assert in CI, throttled warning in prod), then sanitize in
"""Detect NaN (assert when enabled, otherwise a throttled warning), then sanitize in
place: NaN logits (e.g. fp16 activation overflow) are undefined behavior
in sampling kernels and can come back as out-of-vocab token ids. +-1e30
rather than dtype min/max because callers divide logits by temperature,
@@ -90,7 +94,10 @@ def maybe_assert_sum(tensor: torch.Tensor, expected: int, msg: str = "") -> None
def maybe_detect_nan(tensor: Optional[torch.Tensor], msg: str = ""):
"""Async NaN check — no GPU-CPU sync, error surfaces at next sync point."""
if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
if not (
envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get()
or envs.SGLANG_ENABLE_ASYNC_ASSERT.get()
):
return
# A None tensor means there is nothing to probe, e.g. hidden_states on
# capture_hidden_mode=NULL paths (STANDALONE speculative decoding).
+2
View File
@@ -236,6 +236,8 @@ def expect(
(`inv.recover`) is applied unconditionally; only detection is gated. Returns
the recovered value."""
level = resolve_level()
if isinstance(inv.prop, NotNaN) and envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get():
level = InvariantCheckLevel.STRICT
if level >= InvariantCheckLevel.WARN and value is not None and value.numel() > 0:
detail = f"{inv.prop.name}: {msg}" if msg else inv.prop.name
_signal(inv.prop.ok(value), inv=inv, level=level, msg=detail)