Enable async-assert invariant probes by default in CI (#27461)

.github/workflows/_pr-test-stage.yml

@@ -60,6 +60,7 @@ on:
 # defines must be redeclared here for the called job to see the same context.
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
   SGLANG_CUDA_COREDUMP: "1"
   SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
   SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}

.github/workflows/nightly-72-gpu-gb200.yml

@@ -31,6 +31,7 @@ concurrency:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
   SRT_SLURM_BRANCH: sglang-nightly-regression
   SLURM_PARTITION: batch
   SLURM_ACCOUNT: sglang

.github/workflows/nightly-test-nvidia.yml

@@ -46,6 +46,7 @@ concurrency:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
   SGLANG_CUDA_COREDUMP: "1"
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 300

.github/workflows/rerun-test.yml

@@ -59,6 +59,7 @@ on:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
   SGLANG_CUDA_COREDUMP: "1"
   SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
   # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.

.github/workflows/weekly-test-nvidia.yml

@@ -20,6 +20,7 @@ concurrency:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 300
 

python/sglang/test/server_fixtures/eagle_fixture.py

@@ -4,7 +4,6 @@ import time
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils.common import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
@@ -37,21 +36,20 @@ class EagleServerBase(CustomTestCase):
     @classmethod
     def setUpClass(cls):
         cls.base_url = DEFAULT_URL_FOR_TEST
-        with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
+        cls.process = popen_launch_server(
-            cls.process = popen_launch_server(
+            cls.target_model,
-                cls.target_model,
+            cls.base_url,
-                cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=[
-                other_args=[
+                f"--speculative-algorithm={cls.spec_algo}",
-                    f"--speculative-algorithm={cls.spec_algo}",
+                f"--speculative-draft-model-path={cls.draft_model}",
-                    f"--speculative-draft-model-path={cls.draft_model}",
+                f"--speculative-num-steps={cls.spec_steps}",
-                    f"--speculative-num-steps={cls.spec_steps}",
+                f"--speculative-eagle-topk={cls.spec_topk}",
-                    f"--speculative-eagle-topk={cls.spec_topk}",
+                f"--speculative-num-draft-tokens={cls.spec_tokens}",
-                    f"--speculative-num-draft-tokens={cls.spec_tokens}",
+                f"--mem-fraction-static={cls.mem_fraction_static}",
-                    f"--mem-fraction-static={cls.mem_fraction_static}",
+            ]
-                ]
+            + cls.extra_args,
-                + cls.extra_args,
+        )
-            )
 
     @classmethod
     def tearDownClass(cls):

python/sglang/test/server_fixtures/spec_eagle_fixture.py

@@ -122,7 +122,6 @@ class SpecEagleServerBase(CustomTestCase):
         cls.target_model = cls.model
         cls._tokenizer = None
         with contextlib.ExitStack() as stack:
-            stack.enter_context(envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True))
             stack.enter_context(
                 envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True)
             )

scripts/ci/amd/amd_ci_exec.sh

@@ -17,6 +17,7 @@ WORKDIR="/sglang-checkout/test/srt"
 declare -A ENV_MAP=(
   [SGLANG_IS_IN_CI_AMD]=1
   [SGLANG_IS_IN_CI]=1
+  [SGLANG_ENABLE_ASYNC_ASSERT]=1
   [SGLANG_USE_AITER]=1
 )
 

test/registered/spec/dflash/test_dflash.py

@@ -53,10 +53,7 @@ class TestDFlashServerBase(CustomTestCase, MatchedStopMixin, GSM8KMixin):
         old_value = os.environ.get("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN")
         os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] = "1"
         try:
-            with (
+            with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1):
-                envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1),
-                envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True),
-            ):
                 cls.process = popen_launch_server(
                     cls.model,
                     cls.base_url,

test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -49,13 +48,12 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
+        cls.process = popen_launch_server(
-            cls.process = popen_launch_server(
+            cls.model,
-                cls.model,
+            cls.base_url,
-                cls.base_url,
+            timeout=SERVER_LAUNCH_TIMEOUT,
-                timeout=SERVER_LAUNCH_TIMEOUT,
+            other_args=other_args,
-                other_args=other_args,
+        )
-            )
 
     @classmethod
     def tearDownClass(cls):

test/registered/spec/eagle/test_eagle_constrained_decoding.py

@@ -60,10 +60,7 @@ class TestEagleConstrainedDecoding(
             cls.grammar_backend,
         ]
         launch_args.extend(cls.other_launch_args)
-        with (
+        with envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2):
-            envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2),
-            envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True),
-        ):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/spec/eagle/test_eagle_dp_attention.py

@@ -60,10 +60,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
             "--cuda-graph-max-bs",
             "64",
         ]
-        with (
+        with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1):
-            envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True),
-            envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1),
-        ):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -65,13 +64,12 @@ class TestEagleDPAttnServerSmall(CustomTestCase):
             "--speculative-num-draft-tokens",
             "4",
         ]
-        with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
+        cls.process = popen_launch_server(
-            cls.process = popen_launch_server(
+            cls.model,
-                cls.model,
+            cls.base_url,
-                cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
-                other_args=other_args,
+        )
-            )
 
     @classmethod
     def tearDownClass(cls):

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -73,13 +72,12 @@ class TestEagleDPAttnServerLarge(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
+        cls.process = popen_launch_server(
-            cls.process = popen_launch_server(
+            cls.model,
-                cls.model,
+            cls.base_url,
-                cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
-                other_args=other_args,
+        )
-            )
 
     @classmethod
     def tearDownClass(cls):

test/registered/spec/test_constrained_decoding_spec_reasoning.py

@@ -3,7 +3,6 @@ import unittest
 
 import openai
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import (
@@ -51,13 +50,12 @@ class ServerWithGrammar(CustomTestCase):
             "--speculative-num-draft-tokens=8",
         ]
 
-        with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True):
+        cls.process = popen_launch_server(
-            cls.process = popen_launch_server(
+            cls.model,
-                cls.model,
+            cls.base_url,
-                cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=launch_args,
-                other_args=launch_args,
+        )
-            )
 
     @classmethod
     def tearDownClass(cls):

test/registered/spec/test_frozen_kv_mtp.py

@@ -47,7 +47,6 @@ class TestFrozenKVMTP(CustomTestCase):
     def _server_env(cls) -> dict[str, str]:
         env = dict(os.environ)
         env["SGLANG_ENABLE_SPEC_V2"] = "0"
-        env["SGLANG_ENABLE_ASYNC_ASSERT"] = "1"
         return env
 
     @classmethod