Use spec v2 by default (#21062)

2026-04-29 13:40:42 -07:00
parent c3ab5bec7d
commit 79dbfe4505
27 changed files with 413 additions and 207 deletions
@@ -6,6 +6,7 @@ from types import SimpleNamespace

 import requests

+from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -58,32 +59,33 @@ class TestAdaptiveSpeculativeServer(CustomTestCase):
            cls.adaptive_config_path = f.name

        try:
-            cls.process = popen_launch_server(
-                cls.model,
-                cls.base_url,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                other_args=[
-                    "--trust-remote-code",
-                    "--attention-backend",
-                    "triton",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-draft-model-path",
-                    cls.draft_model,
-                    "--speculative-num-steps",
-                    "1",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "2",
-                    "--speculative-adaptive",
-                    "--speculative-adaptive-config",
-                    cls.adaptive_config_path,
-                    "--skip-server-warmup",
-                    "--mem-fraction-static",
-                    "0.7",
-                ],
-            )
+            with envs.SGLANG_ENABLE_SPEC_V2.override(False):
+                cls.process = popen_launch_server(
+                    cls.model,
+                    cls.base_url,
+                    timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                    other_args=[
+                        "--trust-remote-code",
+                        "--attention-backend",
+                        "triton",
+                        "--speculative-algorithm",
+                        "EAGLE",
+                        "--speculative-draft-model-path",
+                        cls.draft_model,
+                        "--speculative-num-steps",
+                        "1",
+                        "--speculative-eagle-topk",
+                        "1",
+                        "--speculative-num-draft-tokens",
+                        "2",
+                        "--speculative-adaptive",
+                        "--speculative-adaptive-config",
+                        cls.adaptive_config_path,
+                        "--skip-server-warmup",
+                        "--mem-fraction-static",
+                        "0.7",
+                    ],
+                )
        except Exception:
            os.unlink(cls.adaptive_config_path)
            raise
@@ -49,13 +49,9 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
            "--model-loader-extra-config",
            '{"enable_multithread_load": true,"num_threads": 64}',
        ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
            True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
            cls.process = popen_launch_server(
                cls.model,
                cls.base_url,
@@ -2,6 +2,7 @@ import random
 import unittest

 import sglang as sgl
+from sglang.srt.environ import envs
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import (
@@ -34,6 +35,14 @@ class TestEAGLEEngine(CustomTestCase):
        "accept_len": 3.6,
    }

+    @classmethod
+    def setUpClass(cls):
+        envs.SGLANG_ENABLE_SPEC_V2.set(False)
+
+    @classmethod
+    def tearDownClass(cls):
+        envs.SGLANG_ENABLE_SPEC_V2.clear()
+
    def setUp(self):
        self.prompt = "Today is a sunny day and I like"
        self.sampling_params = {"temperature": 0, "max_new_tokens": 8}
@@ -30,6 +30,11 @@ class TestEAGLEServerBasic(EagleServerBase):

    extra_args = ["--chunked-prefill-size", 128, "--max-running-requests", 8]

+    @classmethod
+    def setUpClass(cls):
+        with envs.SGLANG_ENABLE_SPEC_V2.override(False):
+            super().setUpClass()
+
    # FIXME(lsyin): move the test methods to kits
    def test_request_abort(self):
        concurrency = 4
@@ -63,9 +63,7 @@ class TestEagle3ServerBase(CustomTestCase, MatchedStopMixin):
            *[str(i) for i in range(1, cls.max_running_requests + 1)],
        ]
        launch_args.extend(cls.other_launch_args)
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
-            True
-        ), envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
+        with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
            1
        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
            True
@@ -65,13 +65,9 @@ class TestEagleDPAttnServerSmall(CustomTestCase):
            "--speculative-num-draft-tokens",
            "4",
        ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
            True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
            cls.process = popen_launch_server(
                cls.model,
                cls.base_url,
@@ -73,13 +73,9 @@ class TestEagleDPAttnServerLarge(CustomTestCase):
            "--model-loader-extra-config",
            '{"enable_multithread_load": true,"num_threads": 64}',
        ]
-        with envs.SGLANG_ENABLE_SPEC_V2.override(
+        with envs.SGLANG_SPEC_NAN_DETECTION.override(
            True
-        ), envs.SGLANG_SPEC_NAN_DETECTION.override(
-            True
-        ), envs.SGLANG_SPEC_OOB_DETECTION.override(
-            True
-        ):
+        ), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
            cls.process = popen_launch_server(
                cls.model,
                cls.base_url,
@@ -1,4 +1,3 @@
-import os
 import unittest
 from types import SimpleNamespace

@@ -80,6 +79,7 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
        # please don't do this if you want to make your inference workload faster
        envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
        envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
+        envs.SGLANG_ENABLE_SPEC_V2.set(False)
        model = cls.model
        cls.process = popen_launch_server(
            model,
@@ -91,6 +91,7 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
    @classmethod
    def tearDownClass(cls):
        kill_process_tree(cls.process.pid)
+        envs.SGLANG_ENABLE_SPEC_V2.clear()

    def test_gsm8k(self):
        requests.get(self.base_url + "/flush_cache")
@@ -140,7 +141,6 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
        # please don't do this if you want to make your inference workload faster
        envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
        envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
-        envs.SGLANG_ENABLE_SPEC_V2.set(True)  # Enable Speculative Decoding V2
        model = cls.model
        cls.process = popen_launch_server(
            model,
@@ -152,8 +152,6 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
    @classmethod
    def tearDownClass(cls):
        kill_process_tree(cls.process.pid)
-        if "SGLANG_ENABLE_SPEC_V2" in os.environ:
-            envs.SGLANG_ENABLE_SPEC_V2.set(False)

    def test_gsm8k(self):
        requests.get(self.base_url + "/flush_cache")