Use spec v2 by default (#21062)

python/sglang/srt/environ.py

@@ -458,7 +458,7 @@ class Envs:
     SGLANG_ROPE_CACHE_ALIGN = EnvInt(128)
 
     # Overlap Spec V2
-    SGLANG_ENABLE_SPEC_V2 = EnvBool(False)
+    SGLANG_ENABLE_SPEC_V2 = EnvBool(True)
     SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False)
 
     # Spec Config

python/sglang/srt/server_args.py

@@ -1979,11 +1979,6 @@ class ServerArgs:
                 logger.info(
                     "Enable multi-layer EAGLE speculative decoding for MiMoV2 model."
                 )
-                if not envs.SGLANG_ENABLE_SPEC_V2.get():
-                    envs.SGLANG_ENABLE_SPEC_V2.set(True)
-                    logger.warning(
-                        "Spec v2 is enabled for multi-layer EAGLE speculative decoding."
-                    )
 
             if self.enable_hierarchical_cache:
                 self.swa_full_tokens_ratio = 1.0
@@ -2000,11 +1995,6 @@ class ServerArgs:
                 logger.info(
                     "Enable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model."
                 )
-                if not envs.SGLANG_ENABLE_SPEC_V2.get():
-                    envs.SGLANG_ENABLE_SPEC_V2.set(True)
-                    logger.warning(
-                        "Spec v2 is enabled for multi-layer EAGLE speculative decoding."
-                    )
             if self.enable_hierarchical_cache:
                 self.swa_full_tokens_ratio = 1.0
                 logger.warning(
@@ -3411,26 +3401,29 @@ class ServerArgs:
                     "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
                 )
 
-            if (
+            spec_v1_reason = None
-                self.speculative_algorithm in ["EAGLE", "EAGLE3", "STANDALONE"]
-                and envs.SGLANG_ENABLE_SPEC_V2.get()
-            ):
-                self.disable_overlap_schedule = False
-                logger.warning(
-                    "Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
-                )
             if (
                 self.speculative_eagle_topk is not None
                 and self.speculative_eagle_topk > 1
+                and not self.disable_overlap_schedule
             ):
-                    raise ValueError(
+                self.disable_overlap_schedule = True
-                        "Spec v2 currently only supports topk = 1 for speculative decoding."
+                spec_v1_reason = "spec v2 currently only supports topk = 1"
+            elif (
+                not envs.SGLANG_ENABLE_SPEC_V2.get()
+                and not self.disable_overlap_schedule
+            ):
+                self.disable_overlap_schedule = True
+                spec_v1_reason = "SGLANG_ENABLE_SPEC_V2=False"
+
+            if self.disable_overlap_schedule:
+                logger.warning(
+                    "Spec v1 is used for eagle/eagle3/standalone speculative decoding because %s.",
+                    spec_v1_reason or "overlap schedule is disabled",
                 )
             else:
-                self.disable_overlap_schedule = True
                 logger.warning(
-                    "Overlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. "
+                    "Spec v2 is enabled by default for eagle/eagle3/standalone speculative decoding."
-                    "You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. "
                 )
 
             if self.enable_mixed_chunk:

test/manual/ascend/test_ascend_deepseek_mtp.py

@@ -53,7 +53,6 @@ class TestAscendDeepSeekMTP(CustomTestCase):
         ]
 
         envs.SGLANG_NPU_USE_MLAPO.set(True)
-        envs.SGLANG_ENABLE_SPEC_V2.set(True)
         envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.set(True)
 
     def test_a_gsm8k(self):

test/manual/test_deepseek_v31.py

@@ -50,7 +50,6 @@ class TestDeepseekV31(unittest.TestCase):
                 DEEPSEEK_V31_MODEL_PATH,
                 tp_size=8,
                 extra_args=base_args + mtp_args,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="TP8+MTP",
             ),
         ]

test/manual/test_glm_46_fp8.py

@@ -41,7 +41,6 @@ class TestGLM46FP8(unittest.TestCase):
                 GLM_4_6_FP8_MODEL_PATH,
                 tp_size=8,
                 extra_args=base_args + mtp_args,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="TP8+MTP",
             ),
         ]

test/manual/test_qwen3_235b.py

@@ -52,7 +52,6 @@ class TestQwen3235BFP8(unittest.TestCase):
                 QWEN3_235B_FP8_MODEL_PATH,
                 tp_size=8,
                 extra_args=base_args + eagle3_args,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="TP8+EP2+EAGLE3",
             ),
         ]

test/registered/4-gpu-models/test_qwen35_models.py

@@ -0,0 +1,242 @@
+import unittest
+from types import SimpleNamespace
+
+import requests
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.accuracy_test_runner import AccuracyTestParams
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin
+
+# This eval harness applies the chat_template, which is critical for qwen3.5
+# to get good accuracy on gsm8k
+from sglang.test.run_combined_tests import run_combined_tests
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    ModelLaunchSettings,
+    popen_launch_server,
+)
+
+register_cuda_ci(est_time=768, suite="stage-c-test-4-gpu-b200")
+
+QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
+ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
+
+
+class TestQwen35FP4(CustomTestCase):
+    def test_gsm8k(self):
+        base_args = [
+            "--tp-size",
+            "4",
+            "--chunked-prefill-size",
+            "2048",
+            "--mamba-scheduler-strategy",
+            "extra_buffer",
+            "--mamba-track-interval",
+            "128",
+            "--mamba-ssm-dtype",
+            "bfloat16",
+            "--max-running-requests",
+            "128",
+            "--reasoning-parser",
+            "qwen3",
+            "--attention-backend",
+            "trtllm_mha",
+            "--quantization",
+            "modelopt_fp4",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true,"num_threads": 64}',
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                QWEN35_FP4_MODEL,
+                extra_args=base_args,
+                variant="Triton",
+            ),
+            # TODO: Fix this and re-enable it
+            # ModelLaunchSettings(
+            #     QWEN35_FP4_MODEL,
+            #     extra_args=base_args + ["--linear-attn-decode-backend", "flashinfer"],
+            #     variant="FlashInfer",
+            # ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Qwen3.5-397B-A17B-NVFP4",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k",
+                baseline_accuracy=ACC_THRESHOLDS[QWEN35_FP4_MODEL]["gsm8k"],
+                num_examples=200,
+                num_threads=128,
+                max_tokens=16000,
+                thinking_mode="qwen3",
+                temperature=0.6,
+                top_p=0.95,
+                top_k=20,
+            ),
+        )
+
+
+class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase):
+    reasoning_parser_name = "qwen3"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.model = QWEN35_FP4_MODEL
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.init_reasoning_token_verifier()
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=[
+                "--tp-size",
+                "4",
+                "--chunked-prefill-size",
+                "2048",
+                "--mamba-scheduler-strategy",
+                "extra_buffer",
+                "--mamba-track-interval",
+                "128",
+                "--mamba-ssm-dtype",
+                "bfloat16",
+                "--max-running-requests",
+                "128",
+                "--reasoning-parser",
+                "qwen3",
+                "--attention-backend",
+                "trtllm_mha",
+                "--quantization",
+                "modelopt_fp4",
+                "--speculative-algorithm",
+                "NEXTN",
+                "--speculative-num-steps",
+                "3",
+                "--speculative-eagle-topk",
+                "1",
+                "--speculative-num-draft-tokens",
+                "4",
+                "--mem-fraction-static",
+                "0.8",
+                "--model-loader-extra-config",
+                '{"enable_multithread_load": true,"num_threads": 64}',
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        args = SimpleNamespace(
+            model=self.model,
+            eval_name="gsm8k",
+            num_shots=5,
+            num_examples=200,
+            max_tokens=16000,
+            num_threads=128,
+            repeat=1,
+            temperature=0.6,
+            top_p=0.95,
+            top_k=20,
+            base_url=self.base_url,
+            host="http://127.0.0.1",
+            port=int(self.base_url.split(":")[-1]),
+        )
+        metrics = run_eval(args)
+        print(f"{metrics=}")
+        self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"])
+
+        server_info = requests.get(self.base_url + "/server_info")
+        avg_spec_accept_length = server_info.json()["internal_states"][0][
+            "avg_spec_accept_length"
+        ]
+        print(f"{avg_spec_accept_length=}")
+        self.assertGreater(avg_spec_accept_length, 3.3)
+
+
+class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase):
+    reasoning_parser_name = "qwen3"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.model = QWEN35_FP4_MODEL
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.init_reasoning_token_verifier()
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=[
+                "--tp-size",
+                "4",
+                "--chunked-prefill-size",
+                "2048",
+                "--mamba-scheduler-strategy",
+                "extra_buffer",
+                "--mamba-track-interval",
+                "128",
+                "--mamba-ssm-dtype",
+                "bfloat16",
+                "--max-running-requests",
+                "128",
+                "--reasoning-parser",
+                "qwen3",
+                "--attention-backend",
+                "trtllm_mha",
+                "--quantization",
+                "modelopt_fp4",
+                "--speculative-algorithm",
+                "NEXTN",
+                "--speculative-num-steps",
+                "3",
+                "--speculative-eagle-topk",
+                "1",
+                "--speculative-num-draft-tokens",
+                "4",
+                "--mem-fraction-static",
+                "0.8",
+                "--model-loader-extra-config",
+                '{"enable_multithread_load": true,"num_threads": 64}',
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        args = SimpleNamespace(
+            model=self.model,
+            eval_name="gsm8k",
+            num_shots=5,
+            num_examples=200,
+            max_tokens=16000,
+            num_threads=128,
+            repeat=1,
+            temperature=0.6,
+            top_p=0.95,
+            top_k=20,
+            base_url=self.base_url,
+            host="http://127.0.0.1",
+            port=int(self.base_url.split(":")[-1]),
+        )
+        metrics = run_eval(args)
+        print(f"{metrics=}")
+        self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"])
+
+        server_info = requests.get(self.base_url + "/server_info")
+        avg_spec_accept_length = server_info.json()["internal_states"][0][
+            "avg_spec_accept_length"
+        ]
+        print(f"{avg_spec_accept_length=}")
+        self.assertGreater(avg_spec_accept_length, 3.3)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/4-gpu-models/test_qwen3_next_models_mtp.py

@@ -1,6 +1,5 @@
 import unittest
 
-from sglang.srt.environ import envs
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
 from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
@@ -94,16 +93,6 @@ class TestQwen3NextMTPV2(GSM8KMixin, KLDivergenceMixin, DefaultServerBase):
         "128",
     ]
 
-    @classmethod
-    def setUpClass(cls):
-        envs.SGLANG_ENABLE_SPEC_V2.set(True)
-        super().setUpClass()
-
-    @classmethod
-    def tearDownClass(cls):
-        envs.SGLANG_ENABLE_SPEC_V2.set(False)
-        super().tearDownClass()
-
 
 if __name__ == "__main__":
     unittest.main()

test/registered/8-gpu-models/test_deepseek_v32.py

@@ -68,7 +68,6 @@ class TestDeepseekV32(unittest.TestCase):
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
                 extra_args=BASE_ARGS + DP_ARGS + TOOL_CALL_ARGS + MTP_ARGS,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="DP8+MTP",
             ),
             # Variant: "tp" - Pure TP=8 only
@@ -83,7 +82,6 @@ class TestDeepseekV32(unittest.TestCase):
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
                 extra_args=BASE_ARGS + TP_ARGS + TOOL_CALL_ARGS + MTP_ARGS,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="TP8+MTP",
             ),
         ]

test/registered/8-gpu-models/test_dsa_models_mtp.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -48,7 +47,6 @@ class TestDeepseekV32DPMTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,
@@ -132,7 +130,6 @@ class TestDeepseekV32TPMTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,
@@ -219,7 +216,6 @@ class TestGLM5DPMTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,
@@ -303,7 +299,6 @@ class TestGLM5TPMTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,

test/registered/8-gpu-models/test_gpt_oss_120b.py

@@ -48,7 +48,6 @@ class TestGptOss120B(unittest.TestCase):
             "--speculative-num-draft-tokens=4",
         ]
         eagle3_env = {
-            "SGLANG_ENABLE_SPEC_V2": "1",
             "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1",
         }
 

test/registered/8-gpu-models/test_mistral_large3.py

@@ -73,7 +73,6 @@ class TestMistralLarge3(unittest.TestCase):
                 MISTRAL_LARGE3_FP8_MODEL_PATH,
                 tp_size=8,
                 extra_args=base_args + eagle_args,
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
                 variant="TP8+MTP",
             ),
             # Variant: "nvfp4" - NVFP4 model + TP=8 + trtllm_mla backend

test/registered/8-gpu-models/test_qwen35.py

@@ -57,7 +57,6 @@ class TestQwen35(unittest.TestCase):
                 tp_size=8,
                 extra_args=base_args + dp_args + mtp_args,
                 variant="TP8+DP8+MTP",
-                env={"SGLANG_ENABLE_SPEC_V2": "1"},
             ),
         ]
 

test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py

@@ -88,7 +88,6 @@ class TestDeepseekR1MXFP4MTP(CustomTestCase):
         cls.model = DEEPSEEK_R1_MODEL_PATH
         cls.base_url = DEFAULT_URL_FOR_TEST
 
-        envs.SGLANG_ENABLE_SPEC_V2.set(True)
         envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.set(True)
 
         other_args = [

test/registered/ascend/basic_function/speculative_inference/test_npu_eagle3.py

@@ -63,7 +63,6 @@ class TestNpuEagle3(CustomTestCase):
 
         cls.extra_envs = {
             "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
-            "SGLANG_ENABLE_SPEC_V2": "1",
         }
         os.environ.update(cls.extra_envs)
 

test/registered/cp/test_deepseek_v32_cp_single_node.py

@@ -1,7 +1,6 @@
 import unittest
 from types import SimpleNamespace
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -52,7 +51,6 @@ class TestDeepseekV32CPInSeqSplit(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,
@@ -119,7 +117,6 @@ class TestDeepseekV32CPRoundRobinSplit(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true, "num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,

test/registered/ep/test_deepep_large.py

@@ -3,6 +3,7 @@ from types import SimpleNamespace
 
 import requests
 
+from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -86,6 +87,7 @@ class TestDeepseekMTP(CustomTestCase):
     def setUpClass(cls):
         cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST
         cls.base_url = DEFAULT_URL_FOR_TEST
+        with envs.SGLANG_ENABLE_SPEC_V2.override(False):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/mla/test_flashmla.py

@@ -9,6 +9,7 @@ from types import SimpleNamespace
 import requests
 import torch
 
+from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -97,6 +98,7 @@ class TestFlashMLAMTP(CustomTestCase):
                 ]
             )
         # Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes
+        with envs.SGLANG_ENABLE_SPEC_V2.override(False):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -54,7 +53,6 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,
@@ -145,7 +143,6 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(True):
         cls.process = popen_launch_server(
             cls.model,
             cls.base_url,

test/registered/spec/eagle/test_adaptive_speculative.py

@@ -6,6 +6,7 @@ from types import SimpleNamespace
 
 import requests
 
+from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -58,6 +59,7 @@ class TestAdaptiveSpeculativeServer(CustomTestCase):
             cls.adaptive_config_path = f.name
 
         try:
+            with envs.SGLANG_ENABLE_SPEC_V2.override(False):
                 cls.process = popen_launch_server(
                     cls.model,
                     cls.base_url,

test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py

@@ -49,13 +49,9 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
             True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/spec/eagle/test_eagle_infer_a.py

@@ -2,6 +2,7 @@ import random
 import unittest
 
 import sglang as sgl
+from sglang.srt.environ import envs
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import (
@@ -34,6 +35,14 @@ class TestEAGLEEngine(CustomTestCase):
         "accept_len": 3.6,
     }
 
+    @classmethod
+    def setUpClass(cls):
+        envs.SGLANG_ENABLE_SPEC_V2.set(False)
+
+    @classmethod
+    def tearDownClass(cls):
+        envs.SGLANG_ENABLE_SPEC_V2.clear()
+
     def setUp(self):
         self.prompt = "Today is a sunny day and I like"
         self.sampling_params = {"temperature": 0, "max_new_tokens": 8}

test/registered/spec/eagle/test_eagle_infer_b.py

@@ -30,6 +30,11 @@ class TestEAGLEServerBasic(EagleServerBase):
 
     extra_args = ["--chunked-prefill-size", 128, "--max-running-requests", 8]
 
+    @classmethod
+    def setUpClass(cls):
+        with envs.SGLANG_ENABLE_SPEC_V2.override(False):
+            super().setUpClass()
+
     # FIXME(lsyin): move the test methods to kits
     def test_request_abort(self):
         concurrency = 4

test/registered/spec/eagle/test_eagle_infer_beta.py

@@ -63,9 +63,7 @@ class TestEagle3ServerBase(CustomTestCase, MatchedStopMixin):
             *[str(i) for i in range(1, cls.max_running_requests + 1)],
         ]
         launch_args.extend(cls.other_launch_args)
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
-            True
-        ), envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
             1
         ), envs.SGLANG_SPEC_NAN_DETECTION.override(
             True

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py

@@ -65,13 +65,9 @@ class TestEagleDPAttnServerSmall(CustomTestCase):
             "--speculative-num-draft-tokens",
             "4",
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
             True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py

@@ -73,13 +73,9 @@ class TestEagleDPAttnServerLarge(CustomTestCase):
             "--model-loader-extra-config",
             '{"enable_multithread_load": true,"num_threads": 64}',
         ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
             True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
             cls.process = popen_launch_server(
                 cls.model,
                 cls.base_url,

test/registered/spec/test_standalone_speculative_decoding.py

@@ -1,4 +1,3 @@
-import os
 import unittest
 from types import SimpleNamespace
 
@@ -80,6 +79,7 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
         # please don't do this if you want to make your inference workload faster
         envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
         envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
+        envs.SGLANG_ENABLE_SPEC_V2.set(False)
         model = cls.model
         cls.process = popen_launch_server(
             model,
@@ -91,6 +91,7 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)
+        envs.SGLANG_ENABLE_SPEC_V2.clear()
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")
@@ -140,7 +141,6 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
         # please don't do this if you want to make your inference workload faster
         envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
         envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
-        envs.SGLANG_ENABLE_SPEC_V2.set(True)  # Enable Speculative Decoding V2
         model = cls.model
         cls.process = popen_launch_server(
             model,
@@ -152,8 +152,6 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)
-        if "SGLANG_ENABLE_SPEC_V2" in os.environ:
-            envs.SGLANG_ENABLE_SPEC_V2.set(False)
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")