[CI] slash handler: lookup runs_on from runner_configs.yml (#25394)
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.accuracy_test_runner import AccuracyTestParams
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
# This eval harness applies the chat_template, which is critical for qwen3.5
|
||||
# to get good accuracy on gsm8k
|
||||
@@ -11,8 +10,6 @@ from sglang.test.test_utils import (
|
||||
ModelLaunchSettings,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200")
|
||||
|
||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
|
||||
from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
|
||||
register_cuda_ci(est_time=142, suite="stage-c-test-4-gpu-h100")
|
||||
|
||||
QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||
from sglang.test.test_utils import (
|
||||
@@ -15,8 +14,6 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=301, suite="stage-c-test-8-gpu-h200")
|
||||
|
||||
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||
from sglang.test.test_utils import (
|
||||
@@ -14,8 +13,6 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=1047, suite="stage-c-test-8-gpu-h200")
|
||||
|
||||
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
|
||||
GLM5_MODEL_PATH = "zai-org/GLM-5-FP8"
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import get_device_sm, kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||
@@ -20,8 +19,6 @@ from sglang.test.test_utils import (
|
||||
|
||||
# FlashAttention3 integration tests (requires SM 90+ / H100)
|
||||
# Multiple test classes: FA3, FA3+MLA, FA3+SpecDecode variants
|
||||
register_cuda_ci(est_time=551, suite="stage-b-test-1-gpu-large")
|
||||
|
||||
GSM_DATASET_PATH = None
|
||||
|
||||
# In case of some machine lack internet connection, we can set OFFLINE_MODE to True.
|
||||
|
||||
@@ -5,7 +5,6 @@ from types import SimpleNamespace
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import get_device_sm, kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION,
|
||||
@@ -15,10 +14,8 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
# Local attention with FA3 (requires SM 90+ / H100, tp=4)
|
||||
register_cuda_ci(est_time=217, suite="stage-c-test-4-gpu-h100")
|
||||
|
||||
|
||||
@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher")
|
||||
class TestFlashAttention3LocalAttn(CustomTestCase):
|
||||
model = DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||
|
||||
register_cuda_ci(est_time=408, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=750, suite="stage-b-test-1-gpu-small-amd-mi35x")
|
||||
|
||||
|
||||
class TestGptOss1Gpu(BaseTestGptOss):
|
||||
def test_mxfp4_20b(self):
|
||||
|
||||
@@ -5,7 +5,6 @@ import requests
|
||||
|
||||
from sglang.lang.chat_template import get_chat_template_by_model_path
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
|
||||
from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
|
||||
from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
|
||||
@@ -22,9 +21,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=245, suite="stage-c-test-4-gpu-h100")
|
||||
register_amd_ci(est_time=350, suite="stage-c-test-4-gpu-amd")
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
is_in_amd_ci(),
|
||||
|
||||
@@ -6,7 +6,6 @@ python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
@@ -16,9 +15,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=496, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin):
|
||||
mmlu_score_threshold = 0.70
|
||||
|
||||
@@ -17,7 +17,6 @@ import os
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.lora_utils import (
|
||||
ALL_OTHER_LORA_MODELS,
|
||||
BACKENDS,
|
||||
@@ -29,13 +28,6 @@ from sglang.test.lora_utils import (
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase, is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=224, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(
|
||||
est_time=200,
|
||||
suite="stage-b-test-1-gpu-small-amd",
|
||||
disabled="see https://github.com/sgl-project/sglang/issues/13107",
|
||||
)
|
||||
|
||||
|
||||
class TestLoRABackend(CustomTestCase):
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
@@ -11,11 +10,8 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
# MLA attention test with MGSM evaluation
|
||||
register_cuda_ci(est_time=181, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestMLA(CustomTestCase, MGSMEnMixin):
|
||||
mgsm_en_score_threshold = 0.8
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from types import SimpleNamespace
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import is_cuda, is_hip, kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
@@ -15,14 +14,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
# DeepSeek-V3 MLA tests with torch compile, FA3, and MTP speculative decoding
|
||||
register_cuda_ci(est_time=543, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(
|
||||
est_time=221,
|
||||
suite="stage-b-test-1-gpu-small-amd",
|
||||
disabled="see https://github.com/sgl-project/sglang/issues/12574",
|
||||
)
|
||||
|
||||
|
||||
class TestMLADeepseekV3(CustomTestCase):
|
||||
@classmethod
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import is_blackwell
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
|
||||
register_cuda_ci(est_time=249, suite="stage-b-test-2-gpu-large")
|
||||
|
||||
|
||||
class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, DefaultServerBase):
|
||||
model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
@@ -10,8 +9,6 @@ from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
||||
# GSM8k + MMMU evaluation
|
||||
|
||||
|
||||
register_cuda_ci(est_time=256, suite="stage-b-test-1-gpu-large")
|
||||
|
||||
MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
@@ -13,9 +12,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=108, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(est_time=130, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestQwen2(CustomTestCase):
|
||||
@classmethod
|
||||
|
||||
@@ -14,7 +14,6 @@ import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -22,9 +21,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=204, stage="stage-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=258, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestToolChoiceLlama32(CustomTestCase):
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
@@ -16,9 +15,6 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=95, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=120, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchOneBatch1GPU(CustomTestCase):
|
||||
|
||||
|
||||
@@ -4,16 +4,12 @@ import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner
|
||||
from sglang.test.test_utils import CustomTestCase, is_in_ci
|
||||
|
||||
# Cross encoder model tests
|
||||
|
||||
|
||||
register_cuda_ci(est_time=125, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(est_time=150, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
MODELS = [
|
||||
("cross-encoder/ms-marco-MiniLM-L6-v2", 1, 1e-2),
|
||||
("BAAI/bge-reranker-v2-m3", 1, 1e-2),
|
||||
|
||||
@@ -6,7 +6,6 @@ import unittest
|
||||
import torch
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner
|
||||
from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci
|
||||
|
||||
@@ -29,8 +28,6 @@ from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci
|
||||
# python -m unittest test_encoder_embedding_models.TestEncoderEmbeddingModels.test_prefill_logits
|
||||
|
||||
|
||||
register_cuda_ci(est_time=444, suite="stage-b-test-1-gpu-small")
|
||||
|
||||
MODELS = [("BAAI/bge-small-en", 1, 1e-5), ("BAAI/bge-m3", 1, 1e-5)]
|
||||
|
||||
ATTENTION_BACKEND = ["torch_native", "triton", "flashinfer"]
|
||||
|
||||
@@ -7,7 +7,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST,
|
||||
@@ -17,8 +16,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=99, suite="stage-b-test-1-gpu-large")
|
||||
|
||||
|
||||
class TestAutoRound(CustomTestCase):
|
||||
@classmethod
|
||||
|
||||
@@ -2,7 +2,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||
from sglang.test.test_utils import (
|
||||
@@ -13,8 +12,6 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=874, suite="stage-c-test-4-gpu-b200")
|
||||
|
||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
|
||||
SERVER_LAUNCH_TIMEOUT = 1200
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import is_hip, kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8,
|
||||
@@ -14,9 +13,6 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=351, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=600, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestEvalFP8Accuracy(CustomTestCase):
|
||||
@classmethod
|
||||
|
||||
@@ -4,7 +4,6 @@ import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1,
|
||||
@@ -16,8 +15,6 @@ from sglang.test.test_utils import (
|
||||
write_results_to_json,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=460, suite="stage-b-test-1-gpu-large")
|
||||
|
||||
MODEL_SCORE_THRESHOLDS = {
|
||||
# Baselines observed with gsm8k 5-shot concatenated format via chat API,
|
||||
# which scores lower than reported benchmarks using proper CoT format.
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
@@ -8,9 +7,6 @@ from sglang.test.test_utils import (
|
||||
run_mmlu_test,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=131, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=108, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestNoChunkedPrefill(CustomTestCase):
|
||||
|
||||
|
||||
@@ -6,12 +6,8 @@ python3 test_overlap_schedule.py
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase, run_mmlu_test
|
||||
|
||||
register_cuda_ci(est_time=267, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=275, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestOverlapSchedule(CustomTestCase):
|
||||
def test_no_radix_attention_chunked_prefill(self):
|
||||
|
||||
@@ -4,7 +4,6 @@ from types import SimpleNamespace
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
|
||||
from sglang.test.test_utils import (
|
||||
@@ -12,9 +11,6 @@ from sglang.test.test_utils import (
|
||||
DEFAULT_TARGET_MODEL_EAGLE3,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=88, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small")
|
||||
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user