minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Test] Drop dead and strictly-subsumed CI test registrations (#40264)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-09-18 17:49:13 -07:00
committed by GitHub
parent fa7e83fd09
commit d507accadc
19 changed files with 9 additions and 1167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/attention/unittests/mla/test_trtllm_mla.py → test/manual/test_trtllm_mla.py
-6
View File
@@ -40,12 +40,6 @@ def _supported() -> tuple[bool, str]:
_SUPPORTED, _SKIP_REASON = _supported() _SUPPORTED, _SKIP_REASON = _supported()
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200")
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-large")
@unittest.skipIf(not _SUPPORTED, _SKIP_REASON) @unittest.skipIf(not _SUPPORTED, _SKIP_REASON)
class TestTRTLLMMLAAttentionBackendCorrectness(CustomTestCase): class TestTRTLLMMLAAttentionBackendCorrectness(CustomTestCase):
# trtllm_mla allows page_size in {32, 64} (server_args.py:2790-2794). # trtllm_mla allows page_size in {32, 64} (server_args.py:2790-2794).
test/registered/debug_utils/test_dump_comparator.py
+1 -1
View File
@@ -23,7 +23,7 @@ class TestCalcRelDiff:
def test_zero_vectors(self) -> None: def test_zero_vectors(self) -> None:
z: torch.Tensor = torch.zeros(5) z: torch.Tensor = torch.zeros(5)
result = _calc_rel_diff(z, z) result = _calc_rel_diff(z, z)
assert not torch.isnan(result) or True # should not crash assert not torch.isnan(result)
class TestArgmaxCoord: class TestArgmaxCoord:
test/registered/e2e/models/test_nvidia_nemotron_3_super_bf16.py
-69
View File
@@ -1,69 +0,0 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=178, stage="extra-b", runner_config="8-gpu-h200")
NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
NEMOTRON_3_SUPER_BF16_ARGS = [
"--tp-size",
"8",
"--trust-remote-code",
"--reasoning-parser",
"nemotron_3",
"--tool-call-parser",
"qwen3_coder",
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 50}',
]
class TestNvidiaNemotron3SuperBF16(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = NEMOTRON_3_SUPER_BF16_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=NEMOTRON_3_SUPER_BF16_ARGS,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=200,
max_tokens=16000,
num_threads=200,
repeat=1,
temperature=1.0,
top_p=0.95,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.96)
if __name__ == "__main__":
unittest.main()
test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py
-89
View File
@@ -93,94 +93,5 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase):
self.assertGreater(metrics["score"], 0.92) self.assertGreater(metrics["score"], 0.92)
class TestDummyWithSBO(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = try_cached_model(DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--disable-radix-cache",
"--mem-fraction-static",
"0.05",
"--max-prefill-tokens",
"16384",
"--max-running-requests",
"256",
"--chunked-prefill-size",
"1024",
"--cuda-graph-bs-decode",
"64",
"--tp",
"4",
"--dp",
"4",
"--ep",
"4",
"--moe-dense-tp-size",
"1",
"--enable-dp-attention",
"--nccl-port",
str(NCCL_PORT_BASE + 1),
"--quantization",
"modelopt_fp4",
"--attention-backend",
"trtllm_mla",
"--moe-runner-backend",
"flashinfer_cutedsl",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--deepep-dispatcher-output-dtype",
"bf16",
"--json-model-override-args",
'{"num_hidden_layers": 1, "first_k_dense_replace": 0, "n_routed_experts": 24}',
"--enable-single-batch-overlap",
"--load-format",
"dummy",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env={
**os.environ,
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
"SGLANG_MOE_NVFP4_DISPATCH": "0",
# Dummy random weights legitimately produce NaN logits; turn
# off the CI crash machinery (async assert, coredump on GPU
# exception, crash-time coredump) so NaN is sanitized with a
# warning instead of killing the scheduler.
"SGLANG_ENABLE_ASYNC_ASSERT": "0",
"SGLANG_SANITIZE_NAN_LOGITS": "1",
"SGLANG_CUDA_COREDUMP": "0",
# Already injected into os.environ by the test process when
# SGLANG_CUDA_COREDUMP=1, so it must be overridden explicitly.
"CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "0",
"SGLANG_CUDA_COREDUMP_BEFORE_CRASH": "0",
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=512,
num_shots=0,
)
metrics = run_eval(args)
print(f"Eval accuracy of GSM8K: {metrics=}")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
test/registered/ep/test_deepep_large.py
-61
View File
@@ -20,67 +20,6 @@ register_cuda_ci(est_time=569, stage="extra-b", runner_config="8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
@unittest.skip("Skip for saving ci time")
class TestDeepseek(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--tp",
"8",
"--enable-dp-attention",
"--dp",
"8",
"--moe-dense-tp-size",
"1",
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--moe-runner-backend",
"deep_gemm",
"--enable-two-batch-overlap",
"--ep-num-redundant-experts",
"32",
"--ep-dispatch-algorithm",
"dynamic",
"--eplb-algorithm",
"deepseek",
"--cuda-graph-bs-decode",
"256",
"--max-running-requests",
"2048",
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
],
)
@classmethod
def tearDownClass(cls):
terminate_and_kill_process_tree(cls.process)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1200,
num_threads=1200,
)
metrics = run_eval(args)
print(f"Eval accuracy of GSM8K: {metrics=}")
self.assertGreater(metrics["score"], 0.92)
class TestDeepseekMTP(CustomTestCase): class TestDeepseekMTP(CustomTestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
test/registered/hicache/test_hicache_storage_file_backend.py
-48
View File
@@ -20,12 +20,10 @@ from sglang.benchmark.utils import get_tokenizer
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval from sglang.test.run_eval import run_eval
from sglang.test.test_utils import ( from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST, DEFAULT_URL_FOR_TEST,
CustomTestCase, CustomTestCase,
is_in_ci,
popen_launch_server, popen_launch_server,
terminate_and_kill_process_tree, terminate_and_kill_process_tree,
) )
@@ -226,33 +224,6 @@ class HiCacheStorageBaseMixin:
) )
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase):
"""Page first layout tests for HiCache Storage functionality"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {"--hicache-mem-layout": "page_first"}
return server_args, {}
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase):
"""MLA Model tests for HiCache Storage functionality"""
@classmethod
def _get_model_name(cls):
"""Use MLA model for testing"""
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {"--tp-size": 2}
return server_args, {}
class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase): class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase):
"""Page first direct tests for HiCache Storage functionality""" """Page first direct tests for HiCache Storage functionality"""
@@ -267,25 +238,6 @@ class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCas
return server_args, {} return server_args, {}
class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase):
"""Accuracy tests for HiCache Storage functionality"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {
"--tp-size": 2,
"--hicache-ratio": 1.5,
}
return server_args, {}
@unittest.skipIf(is_in_ci(), "To skip flaky test")
def test_eval_accuracy(self):
"""Test eval accuracy with cache persistence across cache flushes"""
run_eval_accuracy_test(self)
def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03): def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
"""Generic eval accuracy test with configurable accuracy threshold """Generic eval accuracy test with configurable accuracy threshold
test/registered/hicache/test_hicache_storage_mooncake_backend.py
-32
View File
@@ -18,7 +18,6 @@ from sglang.test.test_utils import (
CustomTestCase, CustomTestCase,
find_available_port, find_available_port,
get_gpu_count, get_gpu_count,
is_in_ci,
) )
register_cuda_ci(est_time=391, stage="base-b", runner_config="2-gpu-large") register_cuda_ci(est_time=391, stage="base-b", runner_config="2-gpu-large")
@@ -211,37 +210,6 @@ class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin):
return server_args, env_vars return server_args, env_vars
'''
# Same as #10131, layer first layout test TODO(mateng): will make it work
class TestMooncakeBackendLayerFirstLayout(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""Layer first layout tests for HiCache-Mooncake backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "layer_first"
server_args["--hicache-io-backend"] = "direct"
return server_args, env_vars
'''
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestMooncakeBackendPageFirstLayout(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""Page first layout tests for HiCache-Mooncake backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "page_first"
return server_args, env_vars
class TestMooncakeBackendMLAModel( class TestMooncakeBackendMLAModel(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
): ):
test/registered/input_embedding/test_input_embeddings.py
-160
View File
@@ -1,160 +0,0 @@
import json
import os
import tempfile
import unittest
import requests
from transformers import AutoModelForCausalLM, AutoTokenizer
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import (
register_amd_ci,
register_cpu_ci,
register_cuda_ci,
)
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=53, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=38, suite="stage-b-test-1-gpu-small-amd")
register_cpu_ci(est_time=58, suite="stage-b-test-cpu-intel")
class TestInputEmbeds(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.tokenizer = AutoTokenizer.from_pretrained(cls.model)
cls.ref_model = AutoModelForCausalLM.from_pretrained(cls.model)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--disable-radix", "--cuda-graph-max-bs-decode", 4],
)
cls.texts = [
"The capital of France is",
"What is the best time of year to visit Japan for cherry blossoms?",
]
def generate_input_embeddings(self, text):
"""Generate input embeddings for a given text."""
input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"]
embeddings = self.ref_model.get_input_embeddings()(input_ids)
return embeddings.squeeze().tolist() # Convert tensor to a list for API use
def send_request(self, payload):
"""Send a POST request to the /generate endpoint and return the response."""
response = requests.post(
self.base_url + "/generate",
json=payload,
timeout=30, # Set a reasonable timeout for the API request
)
if response.status_code == 200:
return response.json()
return {
"error": f"Request failed with status {response.status_code}: {response.text}"
}
def send_file_request(self, file_path):
"""Send a POST request to the /generate_from_file endpoint with a file."""
with open(file_path, "rb") as f:
response = requests.post(
self.base_url + "/generate_from_file",
files={"file": f},
timeout=30, # Set a reasonable timeout for the API request
)
if response.status_code == 200:
return response.json()
return {
"error": f"Request failed with status {response.status_code}: {response.text}"
}
def test_text_based_response(self):
"""Test and print API responses using text-based input."""
for text in self.texts:
payload = {
"model": self.model,
"text": text,
"sampling_params": {"temperature": 0, "max_new_tokens": 50},
}
response = self.send_request(payload)
print(
f"Text Input: {text}\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}"
)
def test_embedding_based_response(self):
"""Test and print API responses using input embeddings."""
for text in self.texts:
embeddings = self.generate_input_embeddings(text)
payload = {
"model": self.model,
"input_embeds": embeddings,
"sampling_params": {"temperature": 0, "max_new_tokens": 50},
}
response = self.send_request(payload)
print(
f"Embeddings Input (for text '{text}'):\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}"
)
def test_compare_text_vs_embedding(self):
"""Test and compare responses for text-based and embedding-based inputs."""
for text in self.texts:
# Text-based payload
text_payload = {
"model": self.model,
"text": text,
"sampling_params": {"temperature": 0, "max_new_tokens": 50},
}
# Embedding-based payload
embeddings = self.generate_input_embeddings(text)
embed_payload = {
"model": self.model,
"input_embeds": embeddings,
"sampling_params": {"temperature": 0, "max_new_tokens": 50},
}
# Get responses
text_response = self.send_request(text_payload)
embed_response = self.send_request(embed_payload)
# Print responses
print(
f"Text Input: {text}\nText-Based Response: {json.dumps(text_response, indent=2)}\n"
)
print(
f"Embeddings Input (for text '{text}'):\nEmbedding-Based Response: {json.dumps(embed_response, indent=2)}\n{'-' * 80}"
)
# This is flaky, so we skip this temporarily
# self.assertEqual(text_response["text"], embed_response["text"])
def test_generate_from_file(self):
"""Test the /generate_from_file endpoint using tokenized embeddings."""
for text in self.texts:
embeddings = self.generate_input_embeddings(text)
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json", delete=False
) as tmp_file:
json.dump(embeddings, tmp_file)
tmp_file_path = tmp_file.name
try:
response = self.send_file_request(tmp_file_path)
print(
f"Text Input: {text}\nResponse from /generate_from_file: {json.dumps(response, indent=2)}\n{'-' * 80}"
)
finally:
# Ensure the temporary file is deleted
os.remove(tmp_file_path)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
if __name__ == "__main__":
unittest.main()
test/registered/kernels/benchmark/diffusion/bench_group_norm_silu.py
+1 -2
View File
@@ -11,7 +11,7 @@ import torch.nn.functional as F
import triton.testing import triton.testing
from sglang.kernels.ops.diffusion import triton_group_norm_silu from sglang.kernels.ops.diffusion import triton_group_norm_silu
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
from sglang.utils import is_in_ci from sglang.utils import is_in_ci
register_cuda_ci( register_cuda_ci(
@@ -20,7 +20,6 @@ register_cuda_ci(
runner_config="1-gpu-large", runner_config="1-gpu-large",
disabled="standalone benchmark", disabled="standalone benchmark",
) )
register_amd_ci(est_time=45, stage="jit-kernel-benchmark", runner_config="amd")
DEVICE = "cuda" DEVICE = "cuda"
EPS = 1e-5 EPS = 1e-5
test/registered/kernels/benchmark/diffusion/bench_norm_impls.py
+1 -2
View File
@@ -22,7 +22,7 @@ from sglang.kernels.ops.diffusion import (
) )
from sglang.kernels.ops.layernorm.norm import fused_add_rmsnorm as jit_fused_add_rmsnorm from sglang.kernels.ops.layernorm.norm import fused_add_rmsnorm as jit_fused_add_rmsnorm
from sglang.kernels.ops.layernorm.norm import rmsnorm as jit_rmsnorm from sglang.kernels.ops.layernorm.norm import rmsnorm as jit_rmsnorm
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
from sglang.utils import is_in_ci from sglang.utils import is_in_ci
register_cuda_ci( register_cuda_ci(
@@ -31,7 +31,6 @@ register_cuda_ci(
runner_config="1-gpu-large", runner_config="1-gpu-large",
disabled="self-skips in CI, standalone tool", disabled="self-skips in CI, standalone tool",
) )
register_amd_ci(est_time=120, stage="jit-kernel-benchmark", runner_config="amd")
os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1") os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1")
test/registered/kernels/benchmark/kv_canary/bench_write.py
+4 -3
View File
@@ -28,13 +28,14 @@ from sglang.kernels.ops.kv_canary.verify import (
VerifyOrWriteContext, VerifyOrWriteContext,
) )
from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large") register_cuda_ci(
est_time=20, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
)
# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
# (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly.
register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True)
_X_NAMES = [ _X_NAMES = [
test/registered/lora/test_lora_drainer.py
-7
View File
@@ -10,7 +10,6 @@ from sglang.test.lora_utils import (
CI_MULTI_LORA_MODELS, CI_MULTI_LORA_MODELS,
run_lora_batch_splitting_equivalence_test, run_lora_batch_splitting_equivalence_test,
) )
from sglang.test.test_utils import is_in_ci
register_cuda_ci(est_time=52, stage="extra-a", runner_config="1-gpu-small") register_cuda_ci(est_time=52, stage="extra-a", runner_config="1-gpu-small")
register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd") register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd")
@@ -33,9 +32,6 @@ def make_req(lora_id, wait_queue_entry_time, max_new_tokens, output_len=0):
class TestLoRADrainer(unittest.TestCase): class TestLoRADrainer(unittest.TestCase):
def test_update_draining_marks_adapter(self): def test_update_draining_marks_adapter(self):
if is_in_ci():
return
with mock.patch("time.monotonic", return_value=MOCK_START_TIME): with mock.patch("time.monotonic", return_value=MOCK_START_TIME):
drainer = LoRADrainer( drainer = LoRADrainer(
max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD
@@ -86,9 +82,6 @@ class TestLoRADrainer(unittest.TestCase):
self.assertEqual(drainer.adapter_to_stats["C"].is_draining_for, "D") self.assertEqual(drainer.adapter_to_stats["C"].is_draining_for, "D")
def test_can_schedule_respects_draining_tolerance(self): def test_can_schedule_respects_draining_tolerance(self):
if is_in_ci():
return
with mock.patch("time.monotonic", return_value=MOCK_START_TIME): with mock.patch("time.monotonic", return_value=MOCK_START_TIME):
drainer = LoRADrainer( drainer = LoRADrainer(
max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD
test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py
-146
View File
@@ -1,146 +0,0 @@
# Copyright 2023-2025 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Regression test for Qwen3-30B-A3B-Instruct-2507 LoRA logprob accuracy.
Compares SGLang LoRA logprobs against reference training logprobs from a
pre-computed dataset. The LoRA adapter and reference data are downloaded from:
https://huggingface.co/datasets/yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507
Usage:
python -m unittest test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff
"""
import multiprocessing as mp
import os
import unittest
import torch
from huggingface_hub import snapshot_download
import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=72, stage="extra-b", runner_config="4-gpu-b200")
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507"
LORA_BACKEND = "triton"
MAX_LORA_RANK = 32
TP_SIZE = 4
MOE_RUNNER_BACKEND = "triton"
EXPERTS_SHARED_OUTER_LORAS = True
PREFILL_ATTENTION_BACKEND = "fa4"
DECODE_ATTENTION_BACKEND = "fa4"
KL_THRESHOLD = 6e-3 # it was 5e-3 with KL 4.766e-3. KL now is 5.008e-3.
def kl_v2(a, b):
a = torch.tensor(a) if not torch.is_tensor(a) else a
b = torch.tensor(b) if not torch.is_tensor(b) else b
return (((a - b) ** 2) * 0.5).mean().item()
def get_prompt_logprobs(engine, input_ids, lora_path):
out = engine.generate(
input_ids=input_ids,
sampling_params={"max_new_tokens": 0, "temperature": 0.0},
return_logprob=True,
logprob_start_len=0,
lora_path=lora_path,
)
return [logprob for logprob, _, _ in out["meta_info"]["input_token_logprobs"]][1:]
class TestLoRAQwen3_30B_A3B_Instruct_2507_LogprobDiff(CustomTestCase):
def test_lora_qwen3_30b_a3b_instruct_2507_logprob_accuracy(self):
adapter_path = snapshot_download(
LORA_HF_REPO,
repo_type="dataset",
)
engine = sgl.Engine(
model_path=BASE_MODEL,
tp_size=TP_SIZE,
enable_lora=True,
max_lora_rank=MAX_LORA_RANK,
lora_paths={"my_lora": adapter_path},
lora_backend=LORA_BACKEND,
attention_backend="flashinfer",
flashinfer_allreduce_fusion_backend="trtllm",
moe_runner_backend=MOE_RUNNER_BACKEND,
experts_shared_outer_loras=EXPERTS_SHARED_OUTER_LORAS,
prefill_attention_backend=PREFILL_ATTENTION_BACKEND,
decode_attention_backend=DECODE_ATTENTION_BACKEND,
)
try:
cdata = torch.load(
os.path.join(adapter_path, "compare_sample_train_data.pt"),
weights_only=False,
)
base_logprobs = get_prompt_logprobs(engine, cdata["tokens"], lora_path=None)
logprobs = get_prompt_logprobs(engine, cdata["tokens"], lora_path="my_lora")
base_t = torch.tensor(base_logprobs)
lora_t = torch.tensor(logprobs)
diff = (base_t - lora_t).abs()
print(
f"[VERIFY] base vs lora: mean_diff={diff.mean().item():.6f}, "
f"max_diff={diff.max().item():.6f}, "
f"identical={torch.equal(base_t, lora_t)}"
)
self.assertFalse(
torch.equal(base_t, lora_t),
"LoRA logprobs should differ from base model logprobs",
)
kl_sglang_trainer = kl_v2(cdata["training_logprobs"], logprobs)
kl_orig_trainer = kl_v2(
cdata["training_logprobs"], cdata["sampling_logprobs"]
)
kl_sglang_orig = kl_v2(logprobs, cdata["sampling_logprobs"])
print(f"KL(orig_sampler, trainer) = {kl_orig_trainer:.6e}")
print(f"KL(sglang, trainer) = {kl_sglang_trainer:.6e}")
print(f"KL(sglang, orig_sampler) = {kl_sglang_orig:.6e}")
self.assertLessEqual(
kl_sglang_trainer,
KL_THRESHOLD,
f"KL(sglang, trainer) = {kl_sglang_trainer:.6e} exceeds "
f"threshold {KL_THRESHOLD}",
)
finally:
engine.shutdown()
if __name__ == "__main__":
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
try:
unittest.main(warnings="ignore", verbosity=2)
finally:
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.synchronize()
test/registered/moe/test_moe_ep.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
) )
# Per-commit: TP=2 EP=2 baseline. # Per-commit: TP=2 EP=2 baseline.
# DeepGEMM/FP8 variant moved to test_moe_ep_nightly.py. # DeepGEMM/FP8 variant: test_moe_ep_extra.py (extra-a).
register_cuda_ci(est_time=93, stage="base-b", runner_config="2-gpu-large") register_cuda_ci(est_time=93, stage="base-b", runner_config="2-gpu-large")
test/registered/quant/test_kvfp4_quant_dequant.py
-120
View File
@@ -1,120 +0,0 @@
#!/usr/bin/env python3
import sys
import time
import numpy as np
import pytest
import torch
from sglang.srt.layers.quantization.kvfp4_tensor import FP4MXBlock16KVQuantizeUtil
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-large")
def calculate_accuracy_metrics(
original: torch.Tensor, reconstructed: torch.Tensor
) -> dict[str, float]:
"""Calculate accuracy metrics between original and reconstructed tensors."""
mse = torch.mean((original - reconstructed) ** 2).item()
mae = torch.mean(torch.abs(original - reconstructed)).item()
# PSNR calculation
max_val = torch.max(torch.abs(original)).item()
psnr = 20 * np.log10(max_val / np.sqrt(mse)) if mse > 0 else float("inf")
# Relative error
rel_error = torch.mean(
torch.abs(original - reconstructed) / (torch.abs(original) + 1e-8)
).item()
return {"MSE": mse, "MAE": mae, "PSNR": psnr, "Relative Error": rel_error}
def run_benchmark(m, n, k, num_runs=10) -> dict[str, dict[str, float]]:
"""Run FP8 vs KVFP4 quantization benchmark and return metrics."""
tensor_bf16 = torch.randn(m, n, k, dtype=torch.bfloat16, device="cuda")
# --- FP8 ---
for _ in range(3): # warmup
_ = tensor_bf16 * 2
torch.cuda.synchronize()
start = time.time()
for _ in range(num_runs):
tensor_fp8 = tensor_bf16.to(torch.float8_e4m3fn)
torch.cuda.synchronize()
fp8_quant_time = (time.time() - start) / num_runs
start = time.time()
for _ in range(num_runs):
tensor_fp8_dequant = tensor_fp8.to(torch.bfloat16)
torch.cuda.synchronize()
fp8_dequant_time = (time.time() - start) / num_runs
fp8_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp8_dequant)
# --- KVFP4 ---
tensor_fp4, scale_factors = FP4MXBlock16KVQuantizeUtil.batched_quantize(tensor_bf16)
_ = FP4MXBlock16KVQuantizeUtil.batched_dequantize(tensor_fp4, scale_factors)
start = time.time()
for _ in range(num_runs):
tensor_fp4, scale_factors = FP4MXBlock16KVQuantizeUtil.batched_quantize(
tensor_bf16
)
torch.cuda.synchronize()
fp4_quant_time = (time.time() - start) / num_runs
start = time.time()
for _ in range(num_runs):
tensor_fp4_dequant = FP4MXBlock16KVQuantizeUtil.batched_dequantize(
tensor_fp4, scale_factors
)
torch.cuda.synchronize()
fp4_dequant_time = (time.time() - start) / num_runs
fp4_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp4_dequant)
return {
"fp8": {
"quant_time": fp8_quant_time,
"dequant_time": fp8_dequant_time,
**fp8_metrics,
},
"fp4": {
"quant_time": fp4_quant_time,
"dequant_time": fp4_dequant_time,
**fp4_metrics,
},
}
# default tensor shapes (m, n, k)
# [M, 1, 576]: DeepSeekR1-FP4 MLA
# [M, 8, 64]: gpt-oss-20b MHA
MNK_FACTORS = [
(64, 1, 576),
(512, 1, 576),
(64, 8, 64),
(512, 8, 64),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
def test_kvfp4_quant_dequant(m, n, k):
"""Benchmark FP8 vs KVFP4 for predefined tensor shapes."""
print(f"\n=== Running benchmark for tensor shape: [{m}, {n}, {k}] ===")
results = run_benchmark(m, n, k)
print("FP8:", results["fp8"])
print("FP4:", results["fp4"])
# Basic assertions to make sure metrics are reasonable
assert results["fp4"]["MSE"] < 0.1
assert results["fp8"]["MSE"] < 0.1
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py
-186
View File
@@ -1,186 +0,0 @@
import unittest
from types import SimpleNamespace
from typing import Optional
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(
est_time=720,
stage="extra-a",
runner_config="2-gpu-large",
disabled="FIXME(kpham-sgl): temporary drop due to accuracies issue",
)
MODEL_NAME = "26B-A4B"
TARGET_PATH = "google/gemma-4-26B-A4B-it"
ASSISTANT_PATH = "google/gemma-4-26B-A4B-it-assistant"
TENSOR_PARALLEL_SIZE = 2
TOPKS = (1, 3)
DRAFT_TOKENS_BY_TOPK = {1: 6, 3: 12}
GSM8K_NUM_EXAMPLES = 200
GSM8K_NUM_THREADS = 128
GSM8K_SCORE_MARGIN = 0.03
SERVER_LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
# Calibrated from deterministic-inference GSM8K runs (200 examples, 5-shot,
# greedy, triton, TP=2). With --enable-deterministic-inference the per-topk
# score is reproducible run-to-run (std=0 over N=20): topk=1 -> 0.445,
# topk=3 -> 0.440.
OBSERVED_GSM8K_SCORES = {1: 0.445, 3: 0.440}
GSM8K_SCORE_THRESHOLD = min(OBSERVED_GSM8K_SCORES.values()) - GSM8K_SCORE_MARGIN
ACCEPT_LENGTH_THRESHOLD = 1.5
def get_server_info(base_url: str) -> dict:
response = requests.get(base_url + "/server_info", timeout=10)
response.raise_for_status()
return response.json()
def get_avg_spec_accept_length(base_url: str) -> Optional[float]:
try:
info = get_server_info(base_url)
except Exception:
return None
internal_states = info.get("internal_states") or []
if not internal_states:
return None
value = internal_states[0].get("avg_spec_accept_length")
if value is None:
return None
return float(value)
class TestGemma4MTP26BA4B(CustomTestCase):
base_url = DEFAULT_URL_FOR_TEST
@classmethod
def _common_server_args(cls) -> list[str]:
args = [
"--attention-backend",
"triton",
"--dtype",
"bfloat16",
"--mem-fraction-static",
"0.55",
"--max-running-requests",
"16",
"--context-length",
"2048",
"--max-total-tokens",
"32768",
"--skip-server-warmup",
# Batch-invariant kernels make the GSM8K score reproducible
# run-to-run; without this the topk=3 score swings ~0.33-0.50.
"--enable-deterministic-inference",
]
if TENSOR_PARALLEL_SIZE > 1:
args += ["--tp-size", str(TENSOR_PARALLEL_SIZE)]
return args
@classmethod
def _server_args(cls, topk: int) -> list[str]:
return [
"--speculative-algorithm",
"NEXTN",
"--speculative-draft-model-path",
ASSISTANT_PATH,
"--speculative-num-steps",
"5",
"--speculative-eagle-topk",
str(topk),
"--speculative-num-draft-tokens",
str(DRAFT_TOKENS_BY_TOPK[topk]),
] + cls._common_server_args()
@classmethod
def _gsm8k_args(cls) -> SimpleNamespace:
return SimpleNamespace(
base_url=cls.base_url,
model=TARGET_PATH,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=GSM8K_NUM_EXAMPLES,
num_threads=GSM8K_NUM_THREADS,
num_shots=5,
)
@staticmethod
def _stop_process(process) -> None:
try:
kill_process_tree(process.pid)
except Exception:
pass
def _run_gsm8k_mtp(self, topk: int) -> None:
process = None
try:
process = popen_launch_server(
TARGET_PATH,
self.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=self._server_args(topk),
)
requests.get(self.base_url + "/flush_cache", timeout=30)
server_info = get_server_info(self.base_url)
self.assertEqual(
server_info.get("speculative_eagle_topk"),
topk,
f"{MODEL_NAME}: server did not start with topk={topk}",
)
self.assertFalse(
bool(server_info.get("disable_cuda_graph")),
f"{MODEL_NAME}/topk{topk}: CUDA graph is disabled",
)
metrics = run_eval(self._gsm8k_args())
mtp_score = float(metrics["score"])
avg_accept = get_avg_spec_accept_length(self.base_url)
finally:
if process is not None:
self._stop_process(process)
print(
f"[Gemma4 {MODEL_NAME} topk={topk}] "
f"score={mtp_score:.4f} threshold={GSM8K_SCORE_THRESHOLD:.4f} "
f"avg_spec_accept_length={avg_accept}"
)
if is_in_ci():
write_github_step_summary(
f"### Gemma4 {MODEL_NAME} MTP topk={topk}\n"
f"score={mtp_score:.4f}\n"
f"threshold={GSM8K_SCORE_THRESHOLD:.4f}\n"
f"avg_spec_accept_length={avg_accept}\n"
)
self.assertGreaterEqual(mtp_score, GSM8K_SCORE_THRESHOLD)
self.assertIsNotNone(avg_accept)
self.assertGreaterEqual(
avg_accept,
ACCEPT_LENGTH_THRESHOLD,
f"{MODEL_NAME}/topk{topk}: accept length too low",
)
def test_gsm8k_mtp(self) -> None:
for topk in TOPKS:
with self.subTest(topk=topk):
self._run_gsm8k_mtp(topk)
if __name__ == "__main__":
unittest.main()
test/registered/unit/layers/quantization/test_bf16_splitk_gemm.py
-6
View File
@@ -2,7 +2,6 @@ import pytest
from sglang.srt.environ import envs from sglang.srt.environ import envs
from sglang.srt.layers.quantization.unquant import ( from sglang.srt.layers.quantization.unquant import (
_BF16_SPLITK_TUNED_TACTICS,
Bf16GemmBackend, Bf16GemmBackend,
should_enable_bf16_splitk_gemm, should_enable_bf16_splitk_gemm,
use_bf16_splitk_gemm, use_bf16_splitk_gemm,
@@ -12,11 +11,6 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=11, suite="base-a-test-cpu") register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@pytest.mark.parametrize("m,n,k", _BF16_SPLITK_TUNED_TACTICS)
def test_splitk_selects_tuned_oakhaven_shape(m: int, n: int, k: int):
assert use_bf16_splitk_gemm(m, n, k)
@pytest.mark.parametrize("m", [0, 33, 64]) @pytest.mark.parametrize("m", [0, 33, 64])
@pytest.mark.parametrize("n,k", [(256, 8192), (512, 8192), (2304, 8192), (2560, 8192)]) @pytest.mark.parametrize("n,k", [(256, 8192), (512, 8192), (2304, 8192), (2560, 8192)])
def test_splitk_keeps_large_m_on_existing_path(m: int, n: int, k: int): def test_splitk_keeps_large_m_on_existing_path(m: int, n: int, k: int):
test/registered/utils/test_type_based_dispatcher.py
-227
View File
@@ -1,227 +0,0 @@
# tests/benchmarks/test_type_dispatcher_e2e.py
"""
E2E test for TypeBasedDispatcher optimization.
Tests real-world scenarios with actual request types.
"""
import timeit
import unittest
from sglang.srt.managers.io_struct import SamplingParams
from sglang.test.ci.ci_register import register_amd_ci, register_cpu_ci
from sglang.utils import TypeBasedDispatcher
register_amd_ci(est_time=10, suite="stage-b-test-1-gpu-small-amd")
register_cpu_ci(est_time=6, suite="stage-b-test-cpu-intel")
class TestTypeBasedDispatcher(unittest.TestCase):
"""Unit tests for TypeBasedDispatcher e2e performance."""
def test_type_dispatcher_e2e_performance(self):
"""End-to-end performance test with real request types"""
print("E2E Performance Test for TypeBasedDispatcher")
print("=" * 50)
from sglang.srt.managers.io_struct import (
AbortReq,
BatchTokenizedEmbeddingReqInput,
BatchTokenizedGenerateReqInput,
ClearHiCacheReqInput,
CloseSessionReqInput,
DestroyWeightsUpdateGroupReqInput,
ExpertDistributionReq,
FlushCacheReqInput,
FreezeGCReq,
GetInternalStateReq,
GetWeightsByNameReqInput,
InitWeightsSendGroupForRemoteInstanceReqInput,
InitWeightsUpdateGroupReqInput,
LoadLoRAAdapterReqInput,
OpenSessionReqInput,
ProfileReq,
ReleaseMemoryOccupationReqInput,
ResumeMemoryOccupationReqInput,
RpcReqInput,
SendWeightsToRemoteInstanceReqInput,
SetInternalStateReq,
SlowDownReqInput,
TokenizedEmbeddingReqInput,
TokenizedGenerateReqInput,
UnloadLoRAAdapterReqInput,
UpdateWeightFromDiskReqInput,
UpdateWeightsFromIPCReqInput,
UpdateWeightsFromTensorReqInput,
)
mapping = [
(TokenizedGenerateReqInput, lambda req: "generate_handled"),
(TokenizedEmbeddingReqInput, lambda req: "embedding_handled"),
(BatchTokenizedGenerateReqInput, lambda req: "batch_generate_handled"),
(
BatchTokenizedEmbeddingReqInput,
lambda req: "batch_generate_embedding_handled",
),
(FlushCacheReqInput, lambda req: "flush_cache_handled"),
(ClearHiCacheReqInput, lambda req: "clear_hicache_handled"),
(AbortReq, lambda req: "abort_handled"),
(OpenSessionReqInput, lambda req: "open_session_handled"),
(CloseSessionReqInput, lambda req: "close_session_handled"),
(
UpdateWeightFromDiskReqInput,
lambda req: "update_weights_from_disk_handled",
),
(
InitWeightsUpdateGroupReqInput,
lambda req: "init_weights_update_group_handled",
),
(
DestroyWeightsUpdateGroupReqInput,
lambda req: "destroy_weights_update_group_handled",
),
(
InitWeightsSendGroupForRemoteInstanceReqInput,
lambda req: "init_weights_send_group_for_remote_instance_handled",
),
(
SendWeightsToRemoteInstanceReqInput,
lambda req: "send_weights_to_remote_instance_handled",
),
(
UpdateWeightsFromTensorReqInput,
lambda req: "update_weights_from_tensor_handled",
),
(
UpdateWeightsFromIPCReqInput,
lambda req: "update_weights_from_ipc_handled",
),
(GetWeightsByNameReqInput, lambda req: "get_weights_by_name_handled"),
(
ReleaseMemoryOccupationReqInput,
lambda req: "release_memory_occupation_handled",
),
(
ResumeMemoryOccupationReqInput,
lambda req: "resume_memory_occupation_handled",
),
(SlowDownReqInput, lambda req: "slow_down_handled"),
(ProfileReq, lambda req: "profile_handled"),
(FreezeGCReq, lambda req: "freeze_gc_handled"),
(GetInternalStateReq, lambda req: "get_internal_state_handled"),
(SetInternalStateReq, lambda req: "set_internal_state_handled"),
(RpcReqInput, lambda req: "rpc_request_handled"),
(ExpertDistributionReq, lambda req: "expert_distribution_handled"),
(LoadLoRAAdapterReqInput, lambda req: "load_lora_adapter_handled"),
(UnloadLoRAAdapterReqInput, lambda req: "unload_lora_adapter_handled"),
]
# Create requests that conforms to the real distribution
test_requests = []
test_requests.append(
TokenizedGenerateReqInput(
input_text="",
input_ids=[1, 2],
input_embeds=None,
mm_inputs=dict(),
token_type_ids=None,
sampling_params=SamplingParams(),
return_logprob=False,
logprob_start_len=0,
top_logprobs_num=0,
token_ids_logprob=[1, 2],
stream=False,
)
)
test_requests.append(
TokenizedEmbeddingReqInput(
input_text="",
input_ids=[1, 2],
mm_inputs=dict(),
token_type_ids=[1, 2],
sampling_params=SamplingParams(),
)
)
test_requests.append(
BatchTokenizedGenerateReqInput(
batch=[
TokenizedGenerateReqInput(
input_text="",
input_ids=[1, 2],
input_embeds=None,
mm_inputs=dict(),
token_type_ids=None,
sampling_params=SamplingParams(),
return_logprob=False,
logprob_start_len=0,
top_logprobs_num=0,
token_ids_logprob=[1, 2],
stream=False,
)
]
)
)
test_requests.append(
BatchTokenizedEmbeddingReqInput(
batch=[
TokenizedEmbeddingReqInput(
input_text="",
input_ids=[1, 2],
mm_inputs=dict(),
token_type_ids=[1, 2],
sampling_params=SamplingParams(),
)
]
)
)
test_requests.append(FlushCacheReqInput())
test_requests.append(ClearHiCacheReqInput())
test_requests.append(AbortReq())
test_requests.append(OpenSessionReqInput(capacity_of_str_len=0))
test_requests.append(CloseSessionReqInput(session_id=""))
test_requests.append(UpdateWeightFromDiskReqInput(model_path=""))
test_requests.append(
InitWeightsUpdateGroupReqInput(
master_address="",
master_port=0,
rank_offset=0,
world_size=0,
group_name="",
)
)
test_requests.append(DestroyWeightsUpdateGroupReqInput())
test_requests.append(
InitWeightsSendGroupForRemoteInstanceReqInput(
master_address="", ports="", group_name="", world_size=0, group_rank=0
)
)
test_requests.append(
SendWeightsToRemoteInstanceReqInput(master_address="", ports="")
)
test_requests.append(
UpdateWeightsFromTensorReqInput(serialized_named_tensors=[])
)
test_requests.append(GetWeightsByNameReqInput(name=""))
test_requests.append(ReleaseMemoryOccupationReqInput())
test_requests.append(RpcReqInput(method=""))
dispatcher = TypeBasedDispatcher(mapping)
# test
time_taken = timeit.timeit(
lambda: [dispatcher(req) for req in test_requests],
number=100, # Average of 100 runs
)
print(f"Total requests: {len(test_requests)}")
print(f"Time taken: {time_taken:.4f}s")
print(f"Requests per second: {len(test_requests) * 100 / time_taken:.0f}")
return time_taken
if __name__ == "__main__":
unittest.main()
test/registered/vlm/test_vision_openai_server_a.py
+1 -1
View File
@@ -229,8 +229,8 @@ class TestKimiVLServer(ImageOpenAITestMixin):
"--mem-fraction-static=0.42", "--mem-fraction-static=0.42",
] ]
@unittest.skip("model context length exceeded")
def test_video_images_chat_completion(self): def test_video_images_chat_completion(self):
# model context length exceeded
pass pass