From 032c9efb4603dd0181c1fbc093fbde9d0772c7b6 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 6 Jun 2026 16:23:21 -0700 Subject: [PATCH] Enable async-assert invariant probes by default in CI (#27461) --- .github/workflows/_pr-test-stage.yml | 1 + .github/workflows/nightly-72-gpu-gb200.yml | 1 + .github/workflows/nightly-test-nvidia.yml | 1 + .github/workflows/rerun-test.yml | 1 + .github/workflows/weekly-test-nvidia.yml | 1 + .../test/server_fixtures/eagle_fixture.py | 30 +++++++++---------- .../server_fixtures/spec_eagle_fixture.py | 1 - scripts/ci/amd/amd_ci_exec.sh | 1 + test/registered/spec/dflash/test_dflash.py | 5 +--- .../eagle/test_deepseek_v3_fp4_mtp_small.py | 14 ++++----- .../eagle/test_eagle_constrained_decoding.py | 5 +--- .../spec/eagle/test_eagle_dp_attention.py | 5 +--- .../test_eagle_infer_beta_dp_attention.py | 14 ++++----- ...est_eagle_infer_beta_dp_attention_large.py | 14 ++++----- ...est_constrained_decoding_spec_reasoning.py | 14 ++++----- test/registered/spec/test_frozen_kv_mtp.py | 1 - 16 files changed, 47 insertions(+), 62 deletions(-) diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index a45b69c4f..89d4068a1 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -60,6 +60,7 @@ on: # defines must be redeclared here for the called job to see the same context. env: SGLANG_IS_IN_CI: true + SGLANG_ENABLE_ASYNC_ASSERT: true SGLANG_CUDA_COREDUMP: "1" SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }} diff --git a/.github/workflows/nightly-72-gpu-gb200.yml b/.github/workflows/nightly-72-gpu-gb200.yml index 51ad81fc6..43b93d026 100644 --- a/.github/workflows/nightly-72-gpu-gb200.yml +++ b/.github/workflows/nightly-72-gpu-gb200.yml @@ -31,6 +31,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_ENABLE_ASYNC_ASSERT: true SRT_SLURM_BRANCH: sglang-nightly-regression SLURM_PARTITION: batch SLURM_ACCOUNT: sglang diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 29de8d888..8af99ef02 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -46,6 +46,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_ENABLE_ASYNC_ASSERT: true SGLANG_CUDA_COREDUMP: "1" HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_ETAG_TIMEOUT: 300 diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index dba7dc124..92287e1a6 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -59,6 +59,7 @@ on: env: SGLANG_IS_IN_CI: true + SGLANG_ENABLE_ASYNC_ASSERT: true SGLANG_CUDA_COREDUMP: "1" SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main. diff --git a/.github/workflows/weekly-test-nvidia.yml b/.github/workflows/weekly-test-nvidia.yml index 01769cf02..e89705061 100644 --- a/.github/workflows/weekly-test-nvidia.yml +++ b/.github/workflows/weekly-test-nvidia.yml @@ -20,6 +20,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_ENABLE_ASYNC_ASSERT: true HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_ETAG_TIMEOUT: 300 diff --git a/python/sglang/test/server_fixtures/eagle_fixture.py b/python/sglang/test/server_fixtures/eagle_fixture.py index b628831de..991585202 100644 --- a/python/sglang/test/server_fixtures/eagle_fixture.py +++ b/python/sglang/test/server_fixtures/eagle_fixture.py @@ -4,7 +4,6 @@ import time import requests -from sglang.srt.environ import envs from sglang.srt.utils.common import kill_process_tree from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, @@ -37,21 +36,20 @@ class EagleServerBase(CustomTestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): - cls.process = popen_launch_server( - cls.target_model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - f"--speculative-algorithm={cls.spec_algo}", - f"--speculative-draft-model-path={cls.draft_model}", - f"--speculative-num-steps={cls.spec_steps}", - f"--speculative-eagle-topk={cls.spec_topk}", - f"--speculative-num-draft-tokens={cls.spec_tokens}", - f"--mem-fraction-static={cls.mem_fraction_static}", - ] - + cls.extra_args, - ) + cls.process = popen_launch_server( + cls.target_model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + f"--speculative-algorithm={cls.spec_algo}", + f"--speculative-draft-model-path={cls.draft_model}", + f"--speculative-num-steps={cls.spec_steps}", + f"--speculative-eagle-topk={cls.spec_topk}", + f"--speculative-num-draft-tokens={cls.spec_tokens}", + f"--mem-fraction-static={cls.mem_fraction_static}", + ] + + cls.extra_args, + ) @classmethod def tearDownClass(cls): diff --git a/python/sglang/test/server_fixtures/spec_eagle_fixture.py b/python/sglang/test/server_fixtures/spec_eagle_fixture.py index 0ef6ccf21..f4041f761 100644 --- a/python/sglang/test/server_fixtures/spec_eagle_fixture.py +++ b/python/sglang/test/server_fixtures/spec_eagle_fixture.py @@ -122,7 +122,6 @@ class SpecEagleServerBase(CustomTestCase): cls.target_model = cls.model cls._tokenizer = None with contextlib.ExitStack() as stack: - stack.enter_context(envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True)) stack.enter_context( envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True) ) diff --git a/scripts/ci/amd/amd_ci_exec.sh b/scripts/ci/amd/amd_ci_exec.sh index 7c0ea9439..2deb9c9a0 100755 --- a/scripts/ci/amd/amd_ci_exec.sh +++ b/scripts/ci/amd/amd_ci_exec.sh @@ -17,6 +17,7 @@ WORKDIR="/sglang-checkout/test/srt" declare -A ENV_MAP=( [SGLANG_IS_IN_CI_AMD]=1 [SGLANG_IS_IN_CI]=1 + [SGLANG_ENABLE_ASYNC_ASSERT]=1 [SGLANG_USE_AITER]=1 ) diff --git a/test/registered/spec/dflash/test_dflash.py b/test/registered/spec/dflash/test_dflash.py index 221957861..7ca357085 100644 --- a/test/registered/spec/dflash/test_dflash.py +++ b/test/registered/spec/dflash/test_dflash.py @@ -53,10 +53,7 @@ class TestDFlashServerBase(CustomTestCase, MatchedStopMixin, GSM8KMixin): old_value = os.environ.get("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN") os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] = "1" try: - with ( - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), - envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), - ): + with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py index b6fd25cb0..3f7d9a5ef 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval @@ -49,13 +48,12 @@ class TestDeepseekV3FP4MTP(CustomTestCase): "--model-loader-extra-config", '{"enable_multithread_load": true,"num_threads": 64}', ] - with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) @classmethod def tearDownClass(cls): diff --git a/test/registered/spec/eagle/test_eagle_constrained_decoding.py b/test/registered/spec/eagle/test_eagle_constrained_decoding.py index 267897c5c..f78c30edf 100644 --- a/test/registered/spec/eagle/test_eagle_constrained_decoding.py +++ b/test/registered/spec/eagle/test_eagle_constrained_decoding.py @@ -60,10 +60,7 @@ class TestEagleConstrainedDecoding( cls.grammar_backend, ] launch_args.extend(cls.other_launch_args) - with ( - envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2), - envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), - ): + with envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index 9fa6593c5..352a82244 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -60,10 +60,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): "--cuda-graph-max-bs", "64", ] - with ( - envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), - ): + with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index ebd6cc1bf..4d19a5d08 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval @@ -65,13 +64,12 @@ class TestEagleDPAttnServerSmall(CustomTestCase): "--speculative-num-draft-tokens", "4", ] - with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) @classmethod def tearDownClass(cls): diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py index 8e6c9c1b2..6f7511244 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval @@ -73,13 +72,12 @@ class TestEagleDPAttnServerLarge(CustomTestCase): "--model-loader-extra-config", '{"enable_multithread_load": true,"num_threads": 64}', ] - with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) @classmethod def tearDownClass(cls): diff --git a/test/registered/spec/test_constrained_decoding_spec_reasoning.py b/test/registered/spec/test_constrained_decoding_spec_reasoning.py index add3849cf..97eda0d8c 100644 --- a/test/registered/spec/test_constrained_decoding_spec_reasoning.py +++ b/test/registered/spec/test_constrained_decoding_spec_reasoning.py @@ -3,7 +3,6 @@ import unittest import openai -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import ( @@ -51,13 +50,12 @@ class ServerWithGrammar(CustomTestCase): "--speculative-num-draft-tokens=8", ] - with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=launch_args, - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) @classmethod def tearDownClass(cls): diff --git a/test/registered/spec/test_frozen_kv_mtp.py b/test/registered/spec/test_frozen_kv_mtp.py index c3eb8ff8e..9d573880a 100644 --- a/test/registered/spec/test_frozen_kv_mtp.py +++ b/test/registered/spec/test_frozen_kv_mtp.py @@ -47,7 +47,6 @@ class TestFrozenKVMTP(CustomTestCase): def _server_env(cls) -> dict[str, str]: env = dict(os.environ) env["SGLANG_ENABLE_SPEC_V2"] = "0" - env["SGLANG_ENABLE_ASYNC_ASSERT"] = "1" return env @classmethod