Tiny cleanup the models' name in test_utils (#15920)
This commit is contained in:
@@ -6,8 +6,8 @@ import requests
|
|||||||
|
|
||||||
from sglang.srt.utils.common import kill_process_tree
|
from sglang.srt.utils.common import kill_process_tree
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -24,8 +24,8 @@ PROMPTS = [
|
|||||||
|
|
||||||
|
|
||||||
class EagleServerBase(CustomTestCase):
|
class EagleServerBase(CustomTestCase):
|
||||||
target_model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
target_model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST
|
draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||||
spec_algo = "EAGLE"
|
spec_algo = "EAGLE"
|
||||||
spec_steps = 5
|
spec_steps = 5
|
||||||
spec_topk = 8
|
spec_topk = 8
|
||||||
|
|||||||
@@ -91,18 +91,24 @@ DEFAULT_MODEL_NAME_FOR_TEST_AWQ_INT4 = (
|
|||||||
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
|
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
|
||||||
)
|
)
|
||||||
|
|
||||||
# EAGLE
|
# EAGLE2 algorithm models
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf"
|
DEFAULT_TARGET_MODEL_EAGLE = "meta-llama/Llama-2-7b-chat-hf"
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B"
|
DEFAULT_DRAFT_MODEL_EAGLE = "lmsys/sglang-EAGLE-llama2-chat-7B"
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 = "meta-llama/Llama-3.1-8B-Instruct"
|
|
||||||
DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST = "Qwen/Qwen3-30B-A3B"
|
# EAGLE3 model
|
||||||
DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST = "Tengyunw/qwen3_30b_moe_eagle3"
|
DEFAULT_TARGET_MODEL_EAGLE3 = "meta-llama/Llama-3.1-8B-Instruct"
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"
|
DEFAULT_DRAFT_MODEL_EAGLE3 = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"
|
||||||
DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST = (
|
|
||||||
"meta-llama/Llama-3.1-8B-Instruct"
|
# EAGLE2 with DP-Attention models
|
||||||
)
|
DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN = "Qwen/Qwen3-30B-A3B"
|
||||||
DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
|
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN = "Tengyunw/qwen3_30b_moe_eagle3"
|
||||||
DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST = "Qwen/Qwen2.5-Coder-7B-Instruct"
|
|
||||||
|
# Standalone speculative decoding models
|
||||||
|
DEFAULT_TARGET_MODEL_STANDALONE = "meta-llama/Llama-3.1-8B-Instruct"
|
||||||
|
DEFAULT_DRAFT_MODEL_STANDALONE = "meta-llama/Llama-3.2-1B-Instruct"
|
||||||
|
|
||||||
|
# N-gram speculative decoding models
|
||||||
|
DEFAULT_TARGET_MODEL_NGRAM = "Qwen/Qwen2.5-Coder-7B-Instruct"
|
||||||
|
|
||||||
# Other use cases
|
# Other use cases
|
||||||
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST = (
|
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST = (
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ from sglang.srt.utils import get_device_sm, kill_process_tree
|
|||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
@@ -30,7 +30,7 @@ OFFLINE_MODE = False
|
|||||||
# Change the path below when OFFLINE_MODE is True.
|
# Change the path below when OFFLINE_MODE is True.
|
||||||
OFFLINE_PATH_DICT = {
|
OFFLINE_PATH_DICT = {
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST: "/shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
DEFAULT_MODEL_NAME_FOR_TEST: "/shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3: "/shared/public/elr-models/jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B",
|
DEFAULT_DRAFT_MODEL_EAGLE3: "/shared/public/elr-models/jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA: "/shared/public/sharing/deepseek/dsv3-test/snapshots/",
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA: "/shared/public/sharing/deepseek/dsv3-test/snapshots/",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN: "/shared/public/sharing/deepseek/dsv3-test-NextN/snapshots/",
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN: "/shared/public/sharing/deepseek/dsv3-test-NextN/snapshots/",
|
||||||
GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl",
|
GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl",
|
||||||
@@ -39,9 +39,7 @@ OFFLINE_PATH_DICT = {
|
|||||||
|
|
||||||
if OFFLINE_MODE:
|
if OFFLINE_MODE:
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST]
|
DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST]
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = OFFLINE_PATH_DICT[
|
DEFAULT_DRAFT_MODEL_EAGLE3 = OFFLINE_PATH_DICT[DEFAULT_DRAFT_MODEL_EAGLE3]
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3
|
|
||||||
]
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST_MLA]
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST_MLA]
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = OFFLINE_PATH_DICT[
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = OFFLINE_PATH_DICT[
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN
|
||||||
@@ -154,7 +152,7 @@ class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest):
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE3",
|
"EAGLE3",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"3",
|
"3",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -188,7 +186,7 @@ class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest):
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE3",
|
"EAGLE3",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"5",
|
"5",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
|
|||||||
@@ -8,10 +8,10 @@ from sglang.srt.utils import get_device_sm, kill_process_tree
|
|||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -59,7 +59,7 @@ class TestHybridAttnBackendBase(CustomTestCase):
|
|||||||
envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False),
|
envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False),
|
||||||
):
|
):
|
||||||
if cls.speculative_decode:
|
if cls.speculative_decode:
|
||||||
model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
else:
|
else:
|
||||||
model = cls.model
|
model = cls.model
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
@@ -129,7 +129,7 @@ class TestHybridAttnBackendSpeculativeDecodingPrefillBackend(TestHybridAttnBacke
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"3",
|
"3",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -152,7 +152,7 @@ class TestHybridAttnBackendSpeculativeDecodingDecodeBackend(TestHybridAttnBacken
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"3",
|
"3",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
|||||||
from sglang.test.kits.json_constrained_kit import TestJSONConstrainedMixin
|
from sglang.test.kits.json_constrained_kit import TestJSONConstrainedMixin
|
||||||
from sglang.test.kits.regex_constrained_kit import TestRegexConstrainedMixin
|
from sglang.test.kits.regex_constrained_kit import TestRegexConstrainedMixin
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -27,8 +27,8 @@ class TestEagleConstrainedDecoding(
|
|||||||
spec_draft_tokens = 6
|
spec_draft_tokens = 6
|
||||||
page_size = 1
|
page_size = 1
|
||||||
other_launch_args = []
|
other_launch_args = []
|
||||||
model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST
|
draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||||
grammar_backend = "xgrammar"
|
grammar_backend = "xgrammar"
|
||||||
eagle_v2 = False
|
eagle_v2 = False
|
||||||
|
|
||||||
|
|||||||
@@ -10,11 +10,11 @@ from sglang.srt.utils import kill_process_tree
|
|||||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE3,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -31,8 +31,8 @@ decode_tolerance: float = 5e-2
|
|||||||
|
|
||||||
class TestEAGLEEngine(CustomTestCase):
|
class TestEAGLEEngine(CustomTestCase):
|
||||||
BASE_CONFIG = {
|
BASE_CONFIG = {
|
||||||
"model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
"model_path": DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
"speculative_draft_model_path": DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
"speculative_draft_model_path": DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"speculative_algorithm": "EAGLE",
|
"speculative_algorithm": "EAGLE",
|
||||||
"speculative_num_steps": 5,
|
"speculative_num_steps": 5,
|
||||||
"speculative_eagle_topk": 4,
|
"speculative_eagle_topk": 4,
|
||||||
@@ -133,7 +133,7 @@ class TestEAGLEEngine(CustomTestCase):
|
|||||||
"skip_special_tokens": False,
|
"skip_special_tokens": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST)
|
tokenizer = get_tokenizer(DEFAULT_TARGET_MODEL_EAGLE)
|
||||||
output = engine.generate(prompt, params)["text"]
|
output = engine.generate(prompt, params)["text"]
|
||||||
print(f"{output=}")
|
print(f"{output=}")
|
||||||
|
|
||||||
@@ -187,8 +187,8 @@ class TestEAGLEEngineTokenMap(TestEAGLEEngine):
|
|||||||
|
|
||||||
class TestEAGLE3Engine(TestEAGLEEngine):
|
class TestEAGLE3Engine(TestEAGLEEngine):
|
||||||
BASE_CONFIG = {
|
BASE_CONFIG = {
|
||||||
"model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3,
|
"model_path": DEFAULT_TARGET_MODEL_EAGLE3,
|
||||||
"speculative_draft_model_path": DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
"speculative_draft_model_path": DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
"speculative_algorithm": "EAGLE3",
|
"speculative_algorithm": "EAGLE3",
|
||||||
"speculative_num_steps": 5,
|
"speculative_num_steps": 5,
|
||||||
"speculative_eagle_topk": 16,
|
"speculative_eagle_topk": 16,
|
||||||
@@ -206,8 +206,8 @@ class TestEAGLE3Engine(TestEAGLEEngine):
|
|||||||
|
|
||||||
class TestEAGLERadixCache(CustomTestCase):
|
class TestEAGLERadixCache(CustomTestCase):
|
||||||
BASE_CONFIG = {
|
BASE_CONFIG = {
|
||||||
"model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3,
|
"model_path": DEFAULT_TARGET_MODEL_EAGLE3,
|
||||||
"speculative_draft_model_path": DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
"speculative_draft_model_path": DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
"speculative_algorithm": "EAGLE3",
|
"speculative_algorithm": "EAGLE3",
|
||||||
"speculative_num_steps": 2,
|
"speculative_num_steps": 2,
|
||||||
"speculative_eagle_topk": 2,
|
"speculative_eagle_topk": 2,
|
||||||
@@ -309,14 +309,14 @@ class TestEAGLEDraftExtend(CustomTestCase):
|
|||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
cls.base_url,
|
cls.base_url,
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
other_args=[
|
other_args=[
|
||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
1,
|
1,
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -375,14 +375,14 @@ class TestEAGLEDraftExtendFlashinfer(TestEAGLEDraftExtend):
|
|||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
cls.base_url,
|
cls.base_url,
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
other_args=[
|
other_args=[
|
||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
1,
|
1,
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -404,14 +404,14 @@ class TestEAGLEDraftExtendTriton(TestEAGLEDraftExtend):
|
|||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
cls.base_url,
|
cls.base_url,
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
other_args=[
|
other_args=[
|
||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
1,
|
1,
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
|
|||||||
@@ -15,10 +15,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
|||||||
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval
|
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval
|
||||||
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
|
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
|
||||||
from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
|
from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import DEFAULT_TARGET_MODEL_EAGLE, run_logprob_check
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
|
||||||
run_logprob_check,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cuda_ci(est_time=1100, suite="stage-b-test-small-1-gpu")
|
register_cuda_ci(est_time=1100, suite="stage-b-test-small-1-gpu")
|
||||||
|
|
||||||
@@ -262,7 +259,7 @@ class TestEAGLEServerBasic(EagleServerBase):
|
|||||||
response = requests.post(
|
response = requests.post(
|
||||||
self.base_url + "/v1/chat/completions",
|
self.base_url + "/v1/chat/completions",
|
||||||
json={
|
json={
|
||||||
"model": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
"model": DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
"messages": messages,
|
"messages": messages,
|
||||||
"temperature": 0,
|
"temperature": 0,
|
||||||
"response_format": {"type": "json_object"},
|
"response_format": {"type": "json_object"},
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ from sglang.test.few_shot_gsm8k import run_eval
|
|||||||
from sglang.test.kits.matched_stop_kit import MatchedStopMixin
|
from sglang.test.kits.matched_stop_kit import MatchedStopMixin
|
||||||
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
|
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -27,8 +27,8 @@ class TestEagleServerBase(CustomTestCase, MatchedStopMixin):
|
|||||||
spec_draft_tokens = 6
|
spec_draft_tokens = 6
|
||||||
page_size = 1
|
page_size = 1
|
||||||
other_launch_args = []
|
other_launch_args = []
|
||||||
model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST
|
draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from sglang.srt.utils import kill_process_tree
|
|||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
|
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_NGRAM,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -32,7 +32,7 @@ DEFAULT_SERVER_ARGS = [
|
|||||||
|
|
||||||
|
|
||||||
class TestNgramSpeculativeDecodingBase(GSM8KMixin, CustomTestCase):
|
class TestNgramSpeculativeDecodingBase(GSM8KMixin, CustomTestCase):
|
||||||
model = DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST
|
model = DEFAULT_TARGET_MODEL_NGRAM
|
||||||
base_url = DEFAULT_URL_FOR_TEST
|
base_url = DEFAULT_URL_FOR_TEST
|
||||||
gsm8k_accuracy_thres = 0.79 # derived tests need to override this
|
gsm8k_accuracy_thres = 0.79 # derived tests need to override this
|
||||||
gsm8k_accept_length_thres = 1.8 # derived spec decoding tests need to override this
|
gsm8k_accept_length_thres = 1.8 # derived spec decoding tests need to override this
|
||||||
|
|||||||
@@ -12,10 +12,10 @@ from sglang.bench_serving import get_tokenizer
|
|||||||
from sglang.srt.utils import is_hip, kill_process_tree
|
from sglang.srt.utils import is_hip, kill_process_tree
|
||||||
from sglang.test.run_eval import run_eval
|
from sglang.test.run_eval import run_eval
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3,
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
DEFAULT_TARGET_MODEL_EAGLE3,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -113,7 +113,7 @@ class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin):
|
|||||||
class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
|
class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
|
||||||
"""HiCache with EAGLE speculative decoding tests"""
|
"""HiCache with EAGLE speculative decoding tests"""
|
||||||
|
|
||||||
model_name = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3
|
model_name = DEFAULT_TARGET_MODEL_EAGLE3
|
||||||
needs_tokenizer = True
|
needs_tokenizer = True
|
||||||
hicache_args = [
|
hicache_args = [
|
||||||
"--enable-hierarchical-cache",
|
"--enable-hierarchical-cache",
|
||||||
@@ -124,7 +124,7 @@ class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE3",
|
"EAGLE3",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3,
|
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
2,
|
2,
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import openai
|
|||||||
from sglang.srt.utils import kill_process_tree
|
from sglang.srt.utils import kill_process_tree
|
||||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
|
||||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -259,10 +259,10 @@ class TestOpenAIServerWithEAGLEAndHiddenStatesEnabled(
|
|||||||
):
|
):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
cls.model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
cls.api_key = "sk-123456"
|
cls.api_key = "sk-123456"
|
||||||
cls.speculative_draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST
|
cls.speculative_draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||||
cls.speculative_algorithm = "EAGLE"
|
cls.speculative_algorithm = "EAGLE"
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
cls.model,
|
cls.model,
|
||||||
@@ -272,7 +272,7 @@ class TestOpenAIServerWithEAGLEAndHiddenStatesEnabled(
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
5,
|
5,
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -289,7 +289,7 @@ class TestOpenAIServerWithEAGLEAndHiddenStatesEnabled(
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
cls.base_url += "/v1"
|
cls.base_url += "/v1"
|
||||||
cls.tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST)
|
cls.tokenizer = get_tokenizer(DEFAULT_TARGET_MODEL_EAGLE)
|
||||||
cls.return_hidden_states = [False, True]
|
cls.return_hidden_states = [False, True]
|
||||||
cls.use_list_input = [True, False]
|
cls.use_list_input = [True, False]
|
||||||
cls.parallel_sample_nums = [1]
|
cls.parallel_sample_nums = [1]
|
||||||
|
|||||||
@@ -5,14 +5,14 @@ import unittest
|
|||||||
import requests
|
import requests
|
||||||
|
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
is_in_amd_ci,
|
is_in_amd_ci,
|
||||||
is_in_ci,
|
is_in_ci,
|
||||||
@@ -328,7 +328,7 @@ class TestBenchServing(CustomTestCase):
|
|||||||
|
|
||||||
def test_online_latency_eagle(self):
|
def test_online_latency_eagle(self):
|
||||||
res = run_bench_serving(
|
res = run_bench_serving(
|
||||||
model=DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
model=DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
num_prompts=300,
|
num_prompts=300,
|
||||||
request_rate=8,
|
request_rate=8,
|
||||||
sharegpt_context_len=3072,
|
sharegpt_context_len=3072,
|
||||||
@@ -338,7 +338,7 @@ class TestBenchServing(CustomTestCase):
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"5",
|
"5",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ from sglang.test.server_fixtures.disaggregation_fixture import (
|
|||||||
PDDisaggregationServerBase,
|
PDDisaggregationServerBase,
|
||||||
)
|
)
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||||
DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST,
|
|
||||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||||
|
DEFAULT_TARGET_MODEL_EAGLE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
popen_launch_pd_server,
|
popen_launch_pd_server,
|
||||||
)
|
)
|
||||||
@@ -280,8 +280,8 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
super().setUpClass()
|
super().setUpClass()
|
||||||
cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST
|
cls.model = DEFAULT_TARGET_MODEL_EAGLE
|
||||||
cls.draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST
|
cls.draft_model = DEFAULT_DRAFT_MODEL_EAGLE
|
||||||
cls.spec_args = [
|
cls.spec_args = [
|
||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"EAGLE",
|
"EAGLE",
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import requests
|
|||||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN,
|
||||||
DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -22,7 +22,7 @@ from sglang.test.test_utils import (
|
|||||||
class TestEAGLE3EngineDPAttention(CustomTestCase):
|
class TestEAGLE3EngineDPAttention(CustomTestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.model = DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST
|
cls.model = DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
other_args = [
|
other_args = [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -35,7 +35,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
|
|||||||
"--speculative-num-draft-tokens",
|
"--speculative-num-draft-tokens",
|
||||||
"32",
|
"32",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN,
|
||||||
"--tp-size",
|
"--tp-size",
|
||||||
"2",
|
"2",
|
||||||
"--dp-size",
|
"--dp-size",
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ from sglang.srt.environ import envs
|
|||||||
from sglang.srt.utils import kill_process_tree
|
from sglang.srt.utils import kill_process_tree
|
||||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||||
from sglang.test.test_utils import (
|
from sglang.test.test_utils import (
|
||||||
DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_STANDALONE,
|
||||||
DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST,
|
DEFAULT_TARGET_MODEL_STANDALONE,
|
||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
@@ -26,7 +26,7 @@ DEFAULT_SERVER_ARGS = [
|
|||||||
"--speculative-algorithm",
|
"--speculative-algorithm",
|
||||||
"STANDALONE",
|
"STANDALONE",
|
||||||
"--speculative-draft-model-path",
|
"--speculative-draft-model-path",
|
||||||
DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST,
|
DEFAULT_DRAFT_MODEL_STANDALONE,
|
||||||
"--speculative-num-steps",
|
"--speculative-num-steps",
|
||||||
"4",
|
"4",
|
||||||
"--speculative-eagle-topk",
|
"--speculative-eagle-topk",
|
||||||
@@ -40,8 +40,8 @@ DEFAULT_SERVER_ARGS = [
|
|||||||
|
|
||||||
class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
|
class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
|
||||||
|
|
||||||
model = DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST
|
model = DEFAULT_TARGET_MODEL_STANDALONE
|
||||||
draft_model = DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST
|
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
|
||||||
base_url = DEFAULT_URL_FOR_TEST
|
base_url = DEFAULT_URL_FOR_TEST
|
||||||
accuracy_threshold = 0.7 # derived tests need to override this
|
accuracy_threshold = 0.7 # derived tests need to override this
|
||||||
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
|
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
|
||||||
|
|||||||
Reference in New Issue
Block a user