ci: tag-gated nightly migration — foundation + 40 whole-file moves (#24725)
Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Alison Shao
co-authored by
hnyls2002
Liangsheng Yin
hnyls2002
Liangsheng Yin
parent
67096f48bf
commit
ba214ef3d3
@@ -20,8 +20,11 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs)
|
||||
register_cuda_ci(est_time=99, stage="stage-c", runner_config="4-gpu-h100")
|
||||
# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs).
|
||||
# Per-commit EAGLE + DP-attn coverage on CUDA is provided by
|
||||
# test_eagle_infer_beta_dp_attention.py (B200 4-gpu), so this H100 variant
|
||||
# is gated to extra-b only.
|
||||
register_cuda_ci(est_time=99, stage="extra-b", runner_config="4-gpu-h100")
|
||||
register_amd_ci(est_time=200, suite="stage-c-test-4-gpu-amd")
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
CustomTestCase,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=357, stage="stage-b", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=357, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestEAGLEEngine(CustomTestCase):
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
|
||||
|
||||
# Per-commit: Paged backend only.
|
||||
# - FA3 base test archived to test/manual/spec/test_spec_ngram_fa3.py
|
||||
# - Triton + Flashinfer moved to test_spec_ngram_extra.py
|
||||
register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestNgramSpeculativeDecodingPaged(NgramServerBase, GSM8KMixin):
|
||||
attention_backend = "flashinfer"
|
||||
extra_args = ["--page-size", "64"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+9
-82
@@ -2,83 +2,22 @@ import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TARGET_MODEL_NGRAM,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
|
||||
|
||||
register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large")
|
||||
|
||||
GSM_DATASET_PATH = None
|
||||
# Extra: Triton + Flashinfer NGRAM backends. Sibling per-commit file
|
||||
# (test_spec_ngram.py) keeps the Paged variant.
|
||||
register_cuda_ci(est_time=254, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
# Default server arguments shared across all tests
|
||||
DEFAULT_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--speculative-algorithm",
|
||||
"NGRAM",
|
||||
"--speculative-num-draft-tokens",
|
||||
"16",
|
||||
"--mem-fraction-static",
|
||||
0.8,
|
||||
]
|
||||
class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin):
|
||||
attention_backend = "triton"
|
||||
|
||||
|
||||
class TestNgramSpeculativeDecodingBase(GSM8KMixin, CustomTestCase):
|
||||
model = DEFAULT_TARGET_MODEL_NGRAM
|
||||
base_url = DEFAULT_URL_FOR_TEST
|
||||
gsm8k_accuracy_thres = 0.79 # derived tests need to override this
|
||||
gsm8k_accept_length_thres = 1.8 # derived spec decoding tests need to override this
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
"""Return the arguments for the server launch. Override in subclasses."""
|
||||
return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# disable deep gemm precompile to make launch server faster
|
||||
# please don't do this if you want to make your inference workload faster
|
||||
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
|
||||
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
|
||||
model = cls.model
|
||||
cls.process = popen_launch_server(
|
||||
model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.get_server_args(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
|
||||
class TestNgramSpeculativeDecodingTriton(TestNgramSpeculativeDecodingBase):
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"]
|
||||
|
||||
|
||||
class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase):
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS + [
|
||||
"--attention-backend",
|
||||
"flashinfer",
|
||||
"--speculative-ngram-external-sam-budget",
|
||||
"8",
|
||||
]
|
||||
class TestNgramSpeculativeDecodingFlashinfer(NgramServerBase, GSM8KMixin):
|
||||
attention_backend = "flashinfer"
|
||||
extra_args = ["--speculative-ngram-external-sam-budget", "8"]
|
||||
|
||||
def test_output_as_corpus_boosts_accept_length(self):
|
||||
"""Baseline → HTTP add corpus → verify accept length boost."""
|
||||
@@ -147,17 +86,5 @@ class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase):
|
||||
)
|
||||
|
||||
|
||||
class TestNgramSpeculativeDecodingPaged(TestNgramSpeculativeDecodingBase):
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS + [
|
||||
"--attention-backend",
|
||||
"flashinfer",
|
||||
"--page-size",
|
||||
"64",
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,27 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
# V2 standalone speculative decoding tests (FA3, Triton, FlashInfer backends).
|
||||
# Non-V2 backends moved to test_spec_standalone_extra.py.
|
||||
register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingBase(StandaloneServerBase, CustomTestCase):
|
||||
attention_backend = "fa3"
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase):
|
||||
attention_backend = "triton"
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingFlashinfer(
|
||||
StandaloneServerBase, CustomTestCase
|
||||
):
|
||||
attention_backend = "flashinfer"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,34 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
# Non-V2 standalone speculative decoding tests (FA3, Triton, FlashInfer
|
||||
# backends). Sibling V2 classes stay per-commit in test_spec_standalone.py.
|
||||
register_cuda_ci(est_time=406, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingBase(StandaloneServerBase, CustomTestCase):
|
||||
attention_backend = "fa3"
|
||||
speculative_eagle_topk = 2
|
||||
speculative_num_draft_tokens = 7
|
||||
enable_spec_v2 = False
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase):
|
||||
attention_backend = "triton"
|
||||
speculative_eagle_topk = 2
|
||||
speculative_num_draft_tokens = 7
|
||||
enable_spec_v2 = False
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingFlashinfer(StandaloneServerBase, CustomTestCase):
|
||||
attention_backend = "flashinfer"
|
||||
speculative_eagle_topk = 2
|
||||
speculative_num_draft_tokens = 7
|
||||
enable_spec_v2 = False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,223 +0,0 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_STANDALONE,
|
||||
DEFAULT_TARGET_MODEL_STANDALONE,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
# Standalone speculative decoding tests (FA3, Triton, FlashInfer backends)
|
||||
register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large")
|
||||
|
||||
GSM_DATASET_PATH = None
|
||||
|
||||
# Default server arguments shared across all tests
|
||||
DEFAULT_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--speculative-algorithm",
|
||||
"STANDALONE",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_STANDALONE,
|
||||
"--speculative-num-steps",
|
||||
"4",
|
||||
"--speculative-eagle-topk",
|
||||
"2",
|
||||
"--speculative-num-draft-tokens",
|
||||
"7",
|
||||
"--mem-fraction-static",
|
||||
0.7,
|
||||
]
|
||||
|
||||
# Default server arguments for V2 tests
|
||||
DEFAULT_SERVER_ARGS_V2 = [
|
||||
"--trust-remote-code",
|
||||
"--cuda-graph-max-bs",
|
||||
"8",
|
||||
"--speculative-algorithm",
|
||||
"STANDALONE",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_STANDALONE,
|
||||
"--speculative-num-steps",
|
||||
"4",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"5",
|
||||
"--mem-fraction-static",
|
||||
0.7,
|
||||
]
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
|
||||
|
||||
model = DEFAULT_TARGET_MODEL_STANDALONE
|
||||
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
|
||||
base_url = DEFAULT_URL_FOR_TEST
|
||||
accuracy_threshold = 0.69 # derived tests need to override this
|
||||
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
"""Return the arguments for the server launch. Override in subclasses."""
|
||||
return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# disable deep gemm precompile to make launch server faster
|
||||
# please don't do this if you want to make your inference workload faster
|
||||
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
|
||||
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
|
||||
envs.SGLANG_ENABLE_SPEC_V2.set(False)
|
||||
model = cls.model
|
||||
cls.process = popen_launch_server(
|
||||
model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.get_server_args(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
envs.SGLANG_ENABLE_SPEC_V2.clear()
|
||||
|
||||
def test_gsm8k(self):
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=100,
|
||||
num_threads=128,
|
||||
num_shots=4,
|
||||
gsm8k_data_path=GSM_DATASET_PATH,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
# Use the appropriate metric key based on the test class
|
||||
metric_key = "score"
|
||||
self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold)
|
||||
|
||||
server_info = requests.get(self.base_url + "/server_info")
|
||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
||||
"avg_spec_accept_length"
|
||||
]
|
||||
print(f"{avg_spec_accept_length=}")
|
||||
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
|
||||
|
||||
model = DEFAULT_TARGET_MODEL_STANDALONE
|
||||
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
|
||||
base_url = DEFAULT_URL_FOR_TEST
|
||||
accuracy_threshold = 0.69 # derived tests need to override this
|
||||
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
"""Return the arguments for the server launch. Override in subclasses."""
|
||||
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "fa3"]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# disable deep gemm precompile to make launch server faster
|
||||
# please don't do this if you want to make your inference workload faster
|
||||
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
|
||||
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
|
||||
model = cls.model
|
||||
cls.process = popen_launch_server(
|
||||
model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.get_server_args(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_gsm8k(self):
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=100,
|
||||
num_threads=128,
|
||||
num_shots=4,
|
||||
gsm8k_data_path=GSM_DATASET_PATH,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
# Use the appropriate metric key based on the test class
|
||||
metric_key = "score"
|
||||
self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold)
|
||||
|
||||
server_info = requests.get(self.base_url + "/server_info")
|
||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
||||
"avg_spec_accept_length"
|
||||
]
|
||||
print(f"{avg_spec_accept_length=}")
|
||||
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase):
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"]
|
||||
|
||||
|
||||
class TestStandaloneSpeculativeDecodingFlashinfer(
|
||||
TestStandaloneSpeculativeDecodingBase
|
||||
):
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"]
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingTriton(
|
||||
TestStandaloneV2SpeculativeDecodingBase
|
||||
):
|
||||
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "triton"]
|
||||
|
||||
def test_radix_attention(self):
|
||||
run_radix_attention_test(self.base_url)
|
||||
assert self.process.poll() is None
|
||||
|
||||
|
||||
class TestStandaloneV2SpeculativeDecodingFlashinfer(
|
||||
TestStandaloneV2SpeculativeDecodingBase
|
||||
):
|
||||
@classmethod
|
||||
def get_server_args(cls):
|
||||
return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "flashinfer"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user