[Debug] Add opt-in NaN detection before sampling (#42690)
Co-authored-by: Qiaolin-Yu <liin1211@outlook.com>
This commit is contained in:
co-authored by
Qiaolin-Yu
parent
393eaed509
commit
b2cb24995d
@@ -512,6 +512,7 @@ class Envs:
|
||||
# page alignment). Off in prod; tests turn it on to fail-fast on
|
||||
# numerical / index violations instead of getting silent NaN cascades.
|
||||
SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False)
|
||||
SGLANG_ENABLE_NAN_LOGITS_CHECK = EnvBool(False)
|
||||
# Signal level for value/index validity checks (nan/inf/oob/...); see
|
||||
# invariants.py. OFF (prod default) runs only the free data layer, WARN
|
||||
# adds throttled logging, STRICT (CI default) crashes on violations.
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.srt.layers.logprob_processor import (
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.sampling.sampling_params import TOP_K_ALL
|
||||
from sglang.srt.utils.async_probe import sanitize_nan_logits
|
||||
from sglang.srt.utils.async_probe import maybe_detect_nan, sanitize_nan_logits
|
||||
from sglang.srt.utils.common import (
|
||||
get_bool_env_var,
|
||||
is_cuda,
|
||||
@@ -163,6 +163,7 @@ class Sampler(nn.Module):
|
||||
to get the unique seed for each position.
|
||||
"""
|
||||
logits = logits_output.next_token_logits
|
||||
maybe_detect_nan(logits, "sampler: next_token_logits")
|
||||
_trace_e2e_sampler(
|
||||
"forward_enter",
|
||||
logits_shape=tuple(logits.shape),
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Async invariant probes — fire torch._assert_async without CPU sync.
|
||||
|
||||
All probes are gated on SGLANG_ENABLE_ASYNC_ASSERT (default off in prod).
|
||||
When the gate is on, a violation surfaces as an assertion at the next CUDA
|
||||
sync point instead of as a silent NaN cascade or illegal-address crash.
|
||||
NaN probes run when SGLANG_ENABLE_NAN_LOGITS_CHECK or SGLANG_ENABLE_ASYNC_ASSERT
|
||||
is enabled. Other probes require SGLANG_ENABLE_ASYNC_ASSERT. Both flags default
|
||||
off. A violation surfaces at the next CUDA sync point instead of a silent NaN
|
||||
cascade or illegal-address crash.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -55,7 +56,10 @@ _nan_warner = _AsyncNanWarner()
|
||||
def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""):
|
||||
"""Non-fatal counterpart of maybe_detect_nan: throttled sync-free warning
|
||||
instead of crashing. Callers sanitize the tensor themselves."""
|
||||
if envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
|
||||
if (
|
||||
envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get()
|
||||
or envs.SGLANG_ENABLE_ASYNC_ASSERT.get()
|
||||
):
|
||||
# The hard assert path already covers detection.
|
||||
return
|
||||
if tensor is None:
|
||||
@@ -64,7 +68,7 @@ def maybe_warn_nan(tensor: Optional[torch.Tensor], msg: str = ""):
|
||||
|
||||
|
||||
def sanitize_nan_logits(logits: torch.Tensor, msg: str = ""):
|
||||
"""Detect NaN (assert in CI, throttled warning in prod), then sanitize in
|
||||
"""Detect NaN (assert when enabled, otherwise a throttled warning), then sanitize in
|
||||
place: NaN logits (e.g. fp16 activation overflow) are undefined behavior
|
||||
in sampling kernels and can come back as out-of-vocab token ids. +-1e30
|
||||
rather than dtype min/max because callers divide logits by temperature,
|
||||
@@ -90,7 +94,10 @@ def maybe_assert_sum(tensor: torch.Tensor, expected: int, msg: str = "") -> None
|
||||
|
||||
def maybe_detect_nan(tensor: Optional[torch.Tensor], msg: str = ""):
|
||||
"""Async NaN check — no GPU-CPU sync, error surfaces at next sync point."""
|
||||
if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
|
||||
if not (
|
||||
envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get()
|
||||
or envs.SGLANG_ENABLE_ASYNC_ASSERT.get()
|
||||
):
|
||||
return
|
||||
# A None tensor means there is nothing to probe, e.g. hidden_states on
|
||||
# capture_hidden_mode=NULL paths (STANDALONE speculative decoding).
|
||||
|
||||
@@ -236,6 +236,8 @@ def expect(
|
||||
(`inv.recover`) is applied unconditionally; only detection is gated. Returns
|
||||
the recovered value."""
|
||||
level = resolve_level()
|
||||
if isinstance(inv.prop, NotNaN) and envs.SGLANG_ENABLE_NAN_LOGITS_CHECK.get():
|
||||
level = InvariantCheckLevel.STRICT
|
||||
if level >= InvariantCheckLevel.WARN and value is not None and value.numel() > 0:
|
||||
detail = f"{inv.prop.name}: {msg}" if msg else inv.prop.name
|
||||
_signal(inv.prop.ok(value), inv=inv, level=level, msg=detail)
|
||||
|
||||
Reference in New Issue
Block a user