From d507accadc49e75dcd75483b1a671e57c6aab1b0 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 18 Sep 2026 17:49:13 -0700 Subject: [PATCH] [Test] Drop dead and strictly-subsumed CI test registrations (#40264) --- .../mla => manual}/test_trtllm_mla.py | 6 - .../debug_utils/test_dump_comparator.py | 2 +- .../test_nvidia_nemotron_3_super_bf16.py | 69 ------ .../test_deepseek_v3_cutedsl_4gpu.py | 89 ------- test/registered/ep/test_deepep_large.py | 61 ----- .../test_hicache_storage_file_backend.py | 48 ---- .../test_hicache_storage_mooncake_backend.py | 32 --- .../input_embedding/test_input_embeddings.py | 160 ------------ .../diffusion/bench_group_norm_silu.py | 3 +- .../benchmark/diffusion/bench_norm_impls.py | 3 +- .../benchmark/kv_canary/bench_write.py | 7 +- test/registered/lora/test_lora_drainer.py | 7 - ...wen3_30b_a3b_instruct_2507_logprob_diff.py | 146 ----------- test/registered/moe/test_moe_ep.py | 2 +- .../quant/test_kvfp4_quant_dequant.py | 120 --------- .../spec/test_gemma4_mtp_26b_a4b_extra.py | 186 -------------- .../quantization/test_bf16_splitk_gemm.py | 6 - .../utils/test_type_based_dispatcher.py | 227 ------------------ .../vlm/test_vision_openai_server_a.py | 2 +- 19 files changed, 9 insertions(+), 1167 deletions(-) rename test/{registered/attention/unittests/mla => manual}/test_trtllm_mla.py (95%) delete mode 100644 test/registered/e2e/models/test_nvidia_nemotron_3_super_bf16.py delete mode 100644 test/registered/input_embedding/test_input_embeddings.py delete mode 100644 test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py delete mode 100755 test/registered/quant/test_kvfp4_quant_dequant.py delete mode 100644 test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py delete mode 100644 test/registered/utils/test_type_based_dispatcher.py diff --git a/test/registered/attention/unittests/mla/test_trtllm_mla.py b/test/manual/test_trtllm_mla.py similarity index 95% rename from test/registered/attention/unittests/mla/test_trtllm_mla.py rename to test/manual/test_trtllm_mla.py index afb20f068..562f67925 100644 --- a/test/registered/attention/unittests/mla/test_trtllm_mla.py +++ b/test/manual/test_trtllm_mla.py @@ -40,12 +40,6 @@ def _supported() -> tuple[bool, str]: _SUPPORTED, _SKIP_REASON = _supported() -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200") -register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-large") - - @unittest.skipIf(not _SUPPORTED, _SKIP_REASON) class TestTRTLLMMLAAttentionBackendCorrectness(CustomTestCase): # trtllm_mla allows page_size in {32, 64} (server_args.py:2790-2794). diff --git a/test/registered/debug_utils/test_dump_comparator.py b/test/registered/debug_utils/test_dump_comparator.py index 0715e3cc9..826649e2c 100644 --- a/test/registered/debug_utils/test_dump_comparator.py +++ b/test/registered/debug_utils/test_dump_comparator.py @@ -23,7 +23,7 @@ class TestCalcRelDiff: def test_zero_vectors(self) -> None: z: torch.Tensor = torch.zeros(5) result = _calc_rel_diff(z, z) - assert not torch.isnan(result) or True # should not crash + assert not torch.isnan(result) class TestArgmaxCoord: diff --git a/test/registered/e2e/models/test_nvidia_nemotron_3_super_bf16.py b/test/registered/e2e/models/test_nvidia_nemotron_3_super_bf16.py deleted file mode 100644 index 4babd0982..000000000 --- a/test/registered/e2e/models/test_nvidia_nemotron_3_super_bf16.py +++ /dev/null @@ -1,69 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=178, stage="extra-b", runner_config="8-gpu-h200") - -NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" - -NEMOTRON_3_SUPER_BF16_ARGS = [ - "--tp-size", - "8", - "--trust-remote-code", - "--reasoning-parser", - "nemotron_3", - "--tool-call-parser", - "qwen3_coder", - "--disable-radix-cache", - "--model-loader-extra-config", - '{"enable_multithread_load": true, "num_threads": 50}', -] - - -class TestNvidiaNemotron3SuperBF16(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = NEMOTRON_3_SUPER_BF16_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=NEMOTRON_3_SUPER_BF16_ARGS, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - model=self.model, - eval_name="gsm8k", - num_shots=5, - num_examples=200, - max_tokens=16000, - num_threads=200, - repeat=1, - temperature=1.0, - top_p=0.95, - base_url=self.base_url, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual(metrics["score"], 0.96) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py index 46b3e6aaf..26905799a 100644 --- a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py @@ -93,94 +93,5 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): self.assertGreater(metrics["score"], 0.92) -class TestDummyWithSBO(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = try_cached_model(DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST) - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--disable-radix-cache", - "--mem-fraction-static", - "0.05", - "--max-prefill-tokens", - "16384", - "--max-running-requests", - "256", - "--chunked-prefill-size", - "1024", - "--cuda-graph-bs-decode", - "64", - "--tp", - "4", - "--dp", - "4", - "--ep", - "4", - "--moe-dense-tp-size", - "1", - "--enable-dp-attention", - "--nccl-port", - str(NCCL_PORT_BASE + 1), - "--quantization", - "modelopt_fp4", - "--attention-backend", - "trtllm_mla", - "--moe-runner-backend", - "flashinfer_cutedsl", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", - "--deepep-dispatcher-output-dtype", - "bf16", - "--json-model-override-args", - '{"num_hidden_layers": 1, "first_k_dense_replace": 0, "n_routed_experts": 24}', - "--enable-single-batch-overlap", - "--load-format", - "dummy", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - env={ - **os.environ, - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", - "SGLANG_MOE_NVFP4_DISPATCH": "0", - # Dummy random weights legitimately produce NaN logits; turn - # off the CI crash machinery (async assert, coredump on GPU - # exception, crash-time coredump) so NaN is sanitized with a - # warning instead of killing the scheduler. - "SGLANG_ENABLE_ASYNC_ASSERT": "0", - "SGLANG_SANITIZE_NAN_LOGITS": "1", - "SGLANG_CUDA_COREDUMP": "0", - # Already injected into os.environ by the test process when - # SGLANG_CUDA_COREDUMP=1, so it must be overridden explicitly. - "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "0", - "SGLANG_CUDA_COREDUMP_BEFORE_CRASH": "0", - }, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=512, - num_threads=512, - num_shots=0, - ) - metrics = run_eval(args) - print(f"Eval accuracy of GSM8K: {metrics=}") - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index c61392c21..24c68344a 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -20,67 +20,6 @@ register_cuda_ci(est_time=569, stage="extra-b", runner_config="8-gpu-h200") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" -@unittest.skip("Skip for saving ci time") -class TestDeepseek(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--tp", - "8", - "--enable-dp-attention", - "--dp", - "8", - "--moe-dense-tp-size", - "1", - "--enable-dp-lm-head", - "--moe-a2a-backend", - "deepep", - "--moe-runner-backend", - "deep_gemm", - "--enable-two-batch-overlap", - "--ep-num-redundant-experts", - "32", - "--ep-dispatch-algorithm", - "dynamic", - "--eplb-algorithm", - "deepseek", - "--cuda-graph-bs-decode", - "256", - "--max-running-requests", - "2048", - "--disable-radix-cache", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ], - ) - - @classmethod - def tearDownClass(cls): - terminate_and_kill_process_tree(cls.process) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=1200, - num_threads=1200, - ) - metrics = run_eval(args) - print(f"Eval accuracy of GSM8K: {metrics=}") - - self.assertGreater(metrics["score"], 0.92) - - class TestDeepseekMTP(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/hicache/test_hicache_storage_file_backend.py b/test/registered/hicache/test_hicache_storage_file_backend.py index 01d5b6946..5372381c2 100644 --- a/test/registered/hicache/test_hicache_storage_file_backend.py +++ b/test/registered/hicache/test_hicache_storage_file_backend.py @@ -20,12 +20,10 @@ from sglang.benchmark.utils import get_tokenizer from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( - DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, terminate_and_kill_process_tree, ) @@ -226,33 +224,6 @@ class HiCacheStorageBaseMixin: ) -@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") -class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase): - """Page first layout tests for HiCache Storage functionality""" - - @classmethod - def _get_additional_server_args_and_env(cls): - """Get additional server arguments specific to configuration - override in subclasses""" - server_args = {"--hicache-mem-layout": "page_first"} - return server_args, {} - - -@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") -class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase): - """MLA Model tests for HiCache Storage functionality""" - - @classmethod - def _get_model_name(cls): - """Use MLA model for testing""" - return DEFAULT_MLA_MODEL_NAME_FOR_TEST - - @classmethod - def _get_additional_server_args_and_env(cls): - """Get additional server arguments specific to configuration - override in subclasses""" - server_args = {"--tp-size": 2} - return server_args, {} - - class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase): """Page first direct tests for HiCache Storage functionality""" @@ -267,25 +238,6 @@ class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCas return server_args, {} -class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase): - """Accuracy tests for HiCache Storage functionality""" - - @classmethod - def _get_additional_server_args_and_env(cls): - """Get additional server arguments specific to configuration - override in subclasses""" - server_args = { - "--tp-size": 2, - "--hicache-ratio": 1.5, - } - - return server_args, {} - - @unittest.skipIf(is_in_ci(), "To skip flaky test") - def test_eval_accuracy(self): - """Test eval accuracy with cache persistence across cache flushes""" - run_eval_accuracy_test(self) - - def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03): """Generic eval accuracy test with configurable accuracy threshold diff --git a/test/registered/hicache/test_hicache_storage_mooncake_backend.py b/test/registered/hicache/test_hicache_storage_mooncake_backend.py index 68ee795d1..e2e036fd3 100644 --- a/test/registered/hicache/test_hicache_storage_mooncake_backend.py +++ b/test/registered/hicache/test_hicache_storage_mooncake_backend.py @@ -18,7 +18,6 @@ from sglang.test.test_utils import ( CustomTestCase, find_available_port, get_gpu_count, - is_in_ci, ) register_cuda_ci(est_time=391, stage="base-b", runner_config="2-gpu-large") @@ -211,37 +210,6 @@ class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin): return server_args, env_vars -''' -# Same as #10131, layer first layout test TODO(mateng): will make it work -class TestMooncakeBackendLayerFirstLayout( - HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase -): - """Layer first layout tests for HiCache-Mooncake backend""" - - @classmethod - def _get_additional_server_args_and_env(cls): - """Get additional server arguments specific to configuration - override in subclasses""" - server_args, env_vars = super()._get_additional_server_args_and_env() - server_args["--hicache-mem-layout"] = "layer_first" - server_args["--hicache-io-backend"] = "direct" - return server_args, env_vars -''' - - -@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") -class TestMooncakeBackendPageFirstLayout( - HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase -): - """Page first layout tests for HiCache-Mooncake backend""" - - @classmethod - def _get_additional_server_args_and_env(cls): - """Get additional server arguments specific to configuration - override in subclasses""" - server_args, env_vars = super()._get_additional_server_args_and_env() - server_args["--hicache-mem-layout"] = "page_first" - return server_args, env_vars - - class TestMooncakeBackendMLAModel( HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase ): diff --git a/test/registered/input_embedding/test_input_embeddings.py b/test/registered/input_embedding/test_input_embeddings.py deleted file mode 100644 index de904880c..000000000 --- a/test/registered/input_embedding/test_input_embeddings.py +++ /dev/null @@ -1,160 +0,0 @@ -import json -import os -import tempfile -import unittest - -import requests -from transformers import AutoModelForCausalLM, AutoTokenizer - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import ( - register_amd_ci, - register_cpu_ci, - register_cuda_ci, -) -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=53, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=38, suite="stage-b-test-1-gpu-small-amd") -register_cpu_ci(est_time=58, suite="stage-b-test-cpu-intel") - - -class TestInputEmbeds(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.tokenizer = AutoTokenizer.from_pretrained(cls.model) - cls.ref_model = AutoModelForCausalLM.from_pretrained(cls.model) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--disable-radix", "--cuda-graph-max-bs-decode", 4], - ) - cls.texts = [ - "The capital of France is", - "What is the best time of year to visit Japan for cherry blossoms?", - ] - - def generate_input_embeddings(self, text): - """Generate input embeddings for a given text.""" - input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"] - embeddings = self.ref_model.get_input_embeddings()(input_ids) - return embeddings.squeeze().tolist() # Convert tensor to a list for API use - - def send_request(self, payload): - """Send a POST request to the /generate endpoint and return the response.""" - response = requests.post( - self.base_url + "/generate", - json=payload, - timeout=30, # Set a reasonable timeout for the API request - ) - if response.status_code == 200: - return response.json() - return { - "error": f"Request failed with status {response.status_code}: {response.text}" - } - - def send_file_request(self, file_path): - """Send a POST request to the /generate_from_file endpoint with a file.""" - with open(file_path, "rb") as f: - response = requests.post( - self.base_url + "/generate_from_file", - files={"file": f}, - timeout=30, # Set a reasonable timeout for the API request - ) - if response.status_code == 200: - return response.json() - return { - "error": f"Request failed with status {response.status_code}: {response.text}" - } - - def test_text_based_response(self): - """Test and print API responses using text-based input.""" - for text in self.texts: - payload = { - "model": self.model, - "text": text, - "sampling_params": {"temperature": 0, "max_new_tokens": 50}, - } - response = self.send_request(payload) - print( - f"Text Input: {text}\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" - ) - - def test_embedding_based_response(self): - """Test and print API responses using input embeddings.""" - for text in self.texts: - embeddings = self.generate_input_embeddings(text) - payload = { - "model": self.model, - "input_embeds": embeddings, - "sampling_params": {"temperature": 0, "max_new_tokens": 50}, - } - response = self.send_request(payload) - print( - f"Embeddings Input (for text '{text}'):\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" - ) - - def test_compare_text_vs_embedding(self): - """Test and compare responses for text-based and embedding-based inputs.""" - for text in self.texts: - # Text-based payload - text_payload = { - "model": self.model, - "text": text, - "sampling_params": {"temperature": 0, "max_new_tokens": 50}, - } - # Embedding-based payload - embeddings = self.generate_input_embeddings(text) - embed_payload = { - "model": self.model, - "input_embeds": embeddings, - "sampling_params": {"temperature": 0, "max_new_tokens": 50}, - } - # Get responses - text_response = self.send_request(text_payload) - embed_response = self.send_request(embed_payload) - # Print responses - print( - f"Text Input: {text}\nText-Based Response: {json.dumps(text_response, indent=2)}\n" - ) - print( - f"Embeddings Input (for text '{text}'):\nEmbedding-Based Response: {json.dumps(embed_response, indent=2)}\n{'-' * 80}" - ) - # This is flaky, so we skip this temporarily - # self.assertEqual(text_response["text"], embed_response["text"]) - - def test_generate_from_file(self): - """Test the /generate_from_file endpoint using tokenized embeddings.""" - for text in self.texts: - embeddings = self.generate_input_embeddings(text) - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp_file: - json.dump(embeddings, tmp_file) - tmp_file_path = tmp_file.name - - try: - response = self.send_file_request(tmp_file_path) - print( - f"Text Input: {text}\nResponse from /generate_from_file: {json.dumps(response, indent=2)}\n{'-' * 80}" - ) - finally: - # Ensure the temporary file is deleted - os.remove(tmp_file_path) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/benchmark/diffusion/bench_group_norm_silu.py b/test/registered/kernels/benchmark/diffusion/bench_group_norm_silu.py index b60240e9b..a79f4e87f 100644 --- a/test/registered/kernels/benchmark/diffusion/bench_group_norm_silu.py +++ b/test/registered/kernels/benchmark/diffusion/bench_group_norm_silu.py @@ -11,7 +11,7 @@ import torch.nn.functional as F import triton.testing from sglang.kernels.ops.diffusion import triton_group_norm_silu -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci from sglang.utils import is_in_ci register_cuda_ci( @@ -20,7 +20,6 @@ register_cuda_ci( runner_config="1-gpu-large", disabled="standalone benchmark", ) -register_amd_ci(est_time=45, stage="jit-kernel-benchmark", runner_config="amd") DEVICE = "cuda" EPS = 1e-5 diff --git a/test/registered/kernels/benchmark/diffusion/bench_norm_impls.py b/test/registered/kernels/benchmark/diffusion/bench_norm_impls.py index 309c533c8..4990acba5 100644 --- a/test/registered/kernels/benchmark/diffusion/bench_norm_impls.py +++ b/test/registered/kernels/benchmark/diffusion/bench_norm_impls.py @@ -22,7 +22,7 @@ from sglang.kernels.ops.diffusion import ( ) from sglang.kernels.ops.layernorm.norm import fused_add_rmsnorm as jit_fused_add_rmsnorm from sglang.kernels.ops.layernorm.norm import rmsnorm as jit_rmsnorm -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci from sglang.utils import is_in_ci register_cuda_ci( @@ -31,7 +31,6 @@ register_cuda_ci( runner_config="1-gpu-large", disabled="self-skips in CI, standalone tool", ) -register_amd_ci(est_time=120, stage="jit-kernel-benchmark", runner_config="amd") os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1") diff --git a/test/registered/kernels/benchmark/kv_canary/bench_write.py b/test/registered/kernels/benchmark/kv_canary/bench_write.py index da8fdd962..ffe05617e 100644 --- a/test/registered/kernels/benchmark/kv_canary/bench_write.py +++ b/test/registered/kernels/benchmark/kv_canary/bench_write.py @@ -28,13 +28,14 @@ from sglang.kernels.ops.kv_canary.verify import ( VerifyOrWriteContext, ) from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large") +register_cuda_ci( + est_time=20, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" +) # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. -register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True) _X_NAMES = [ diff --git a/test/registered/lora/test_lora_drainer.py b/test/registered/lora/test_lora_drainer.py index 9b4900cbd..ae25ffc2b 100644 --- a/test/registered/lora/test_lora_drainer.py +++ b/test/registered/lora/test_lora_drainer.py @@ -10,7 +10,6 @@ from sglang.test.lora_utils import ( CI_MULTI_LORA_MODELS, run_lora_batch_splitting_equivalence_test, ) -from sglang.test.test_utils import is_in_ci register_cuda_ci(est_time=52, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd") @@ -33,9 +32,6 @@ def make_req(lora_id, wait_queue_entry_time, max_new_tokens, output_len=0): class TestLoRADrainer(unittest.TestCase): def test_update_draining_marks_adapter(self): - if is_in_ci(): - return - with mock.patch("time.monotonic", return_value=MOCK_START_TIME): drainer = LoRADrainer( max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD @@ -86,9 +82,6 @@ class TestLoRADrainer(unittest.TestCase): self.assertEqual(drainer.adapter_to_stats["C"].is_draining_for, "D") def test_can_schedule_respects_draining_tolerance(self): - if is_in_ci(): - return - with mock.patch("time.monotonic", return_value=MOCK_START_TIME): drainer = LoRADrainer( max_loras_per_batch=1, max_wait_time_secs=LORA_DRAIN_WAIT_THRESHOLD diff --git a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py deleted file mode 100644 index 850a91169..000000000 --- a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2023-2025 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -""" -Regression test for Qwen3-30B-A3B-Instruct-2507 LoRA logprob accuracy. - -Compares SGLang LoRA logprobs against reference training logprobs from a -pre-computed dataset. The LoRA adapter and reference data are downloaded from: -https://huggingface.co/datasets/yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507 - -Usage: - python -m unittest test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff -""" - -import multiprocessing as mp -import os -import unittest - -import torch -from huggingface_hub import snapshot_download - -import sglang as sgl -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=72, stage="extra-b", runner_config="4-gpu-b200") - -BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" -LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507" -LORA_BACKEND = "triton" -MAX_LORA_RANK = 32 -TP_SIZE = 4 -MOE_RUNNER_BACKEND = "triton" -EXPERTS_SHARED_OUTER_LORAS = True -PREFILL_ATTENTION_BACKEND = "fa4" -DECODE_ATTENTION_BACKEND = "fa4" - -KL_THRESHOLD = 6e-3 # it was 5e-3 with KL 4.766e-3. KL now is 5.008e-3. - - -def kl_v2(a, b): - a = torch.tensor(a) if not torch.is_tensor(a) else a - b = torch.tensor(b) if not torch.is_tensor(b) else b - return (((a - b) ** 2) * 0.5).mean().item() - - -def get_prompt_logprobs(engine, input_ids, lora_path): - out = engine.generate( - input_ids=input_ids, - sampling_params={"max_new_tokens": 0, "temperature": 0.0}, - return_logprob=True, - logprob_start_len=0, - lora_path=lora_path, - ) - return [logprob for logprob, _, _ in out["meta_info"]["input_token_logprobs"]][1:] - - -class TestLoRAQwen3_30B_A3B_Instruct_2507_LogprobDiff(CustomTestCase): - def test_lora_qwen3_30b_a3b_instruct_2507_logprob_accuracy(self): - adapter_path = snapshot_download( - LORA_HF_REPO, - repo_type="dataset", - ) - - engine = sgl.Engine( - model_path=BASE_MODEL, - tp_size=TP_SIZE, - enable_lora=True, - max_lora_rank=MAX_LORA_RANK, - lora_paths={"my_lora": adapter_path}, - lora_backend=LORA_BACKEND, - attention_backend="flashinfer", - flashinfer_allreduce_fusion_backend="trtllm", - moe_runner_backend=MOE_RUNNER_BACKEND, - experts_shared_outer_loras=EXPERTS_SHARED_OUTER_LORAS, - prefill_attention_backend=PREFILL_ATTENTION_BACKEND, - decode_attention_backend=DECODE_ATTENTION_BACKEND, - ) - - try: - cdata = torch.load( - os.path.join(adapter_path, "compare_sample_train_data.pt"), - weights_only=False, - ) - - base_logprobs = get_prompt_logprobs(engine, cdata["tokens"], lora_path=None) - logprobs = get_prompt_logprobs(engine, cdata["tokens"], lora_path="my_lora") - - base_t = torch.tensor(base_logprobs) - lora_t = torch.tensor(logprobs) - diff = (base_t - lora_t).abs() - print( - f"[VERIFY] base vs lora: mean_diff={diff.mean().item():.6f}, " - f"max_diff={diff.max().item():.6f}, " - f"identical={torch.equal(base_t, lora_t)}" - ) - - self.assertFalse( - torch.equal(base_t, lora_t), - "LoRA logprobs should differ from base model logprobs", - ) - - kl_sglang_trainer = kl_v2(cdata["training_logprobs"], logprobs) - kl_orig_trainer = kl_v2( - cdata["training_logprobs"], cdata["sampling_logprobs"] - ) - kl_sglang_orig = kl_v2(logprobs, cdata["sampling_logprobs"]) - - print(f"KL(orig_sampler, trainer) = {kl_orig_trainer:.6e}") - print(f"KL(sglang, trainer) = {kl_sglang_trainer:.6e}") - print(f"KL(sglang, orig_sampler) = {kl_sglang_orig:.6e}") - - self.assertLessEqual( - kl_sglang_trainer, - KL_THRESHOLD, - f"KL(sglang, trainer) = {kl_sglang_trainer:.6e} exceeds " - f"threshold {KL_THRESHOLD}", - ) - - finally: - engine.shutdown() - - -if __name__ == "__main__": - try: - mp.set_start_method("spawn") - except RuntimeError: - pass - - try: - unittest.main(warnings="ignore", verbosity=2) - finally: - if torch.cuda.is_available(): - torch.cuda.empty_cache() - torch.cuda.synchronize() diff --git a/test/registered/moe/test_moe_ep.py b/test/registered/moe/test_moe_ep.py index 3e7d27b4f..900f98d7f 100644 --- a/test/registered/moe/test_moe_ep.py +++ b/test/registered/moe/test_moe_ep.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( ) # Per-commit: TP=2 EP=2 baseline. -# DeepGEMM/FP8 variant moved to test_moe_ep_nightly.py. +# DeepGEMM/FP8 variant: test_moe_ep_extra.py (extra-a). register_cuda_ci(est_time=93, stage="base-b", runner_config="2-gpu-large") diff --git a/test/registered/quant/test_kvfp4_quant_dequant.py b/test/registered/quant/test_kvfp4_quant_dequant.py deleted file mode 100755 index 1fd0b9be8..000000000 --- a/test/registered/quant/test_kvfp4_quant_dequant.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import time - -import numpy as np -import pytest -import torch - -from sglang.srt.layers.quantization.kvfp4_tensor import FP4MXBlock16KVQuantizeUtil -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-large") - - -def calculate_accuracy_metrics( - original: torch.Tensor, reconstructed: torch.Tensor -) -> dict[str, float]: - """Calculate accuracy metrics between original and reconstructed tensors.""" - mse = torch.mean((original - reconstructed) ** 2).item() - mae = torch.mean(torch.abs(original - reconstructed)).item() - - # PSNR calculation - max_val = torch.max(torch.abs(original)).item() - psnr = 20 * np.log10(max_val / np.sqrt(mse)) if mse > 0 else float("inf") - - # Relative error - rel_error = torch.mean( - torch.abs(original - reconstructed) / (torch.abs(original) + 1e-8) - ).item() - - return {"MSE": mse, "MAE": mae, "PSNR": psnr, "Relative Error": rel_error} - - -def run_benchmark(m, n, k, num_runs=10) -> dict[str, dict[str, float]]: - """Run FP8 vs KVFP4 quantization benchmark and return metrics.""" - tensor_bf16 = torch.randn(m, n, k, dtype=torch.bfloat16, device="cuda") - - # --- FP8 --- - for _ in range(3): # warmup - _ = tensor_bf16 * 2 - torch.cuda.synchronize() - - start = time.time() - for _ in range(num_runs): - tensor_fp8 = tensor_bf16.to(torch.float8_e4m3fn) - torch.cuda.synchronize() - fp8_quant_time = (time.time() - start) / num_runs - - start = time.time() - for _ in range(num_runs): - tensor_fp8_dequant = tensor_fp8.to(torch.bfloat16) - torch.cuda.synchronize() - fp8_dequant_time = (time.time() - start) / num_runs - - fp8_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp8_dequant) - - # --- KVFP4 --- - tensor_fp4, scale_factors = FP4MXBlock16KVQuantizeUtil.batched_quantize(tensor_bf16) - _ = FP4MXBlock16KVQuantizeUtil.batched_dequantize(tensor_fp4, scale_factors) - - start = time.time() - for _ in range(num_runs): - tensor_fp4, scale_factors = FP4MXBlock16KVQuantizeUtil.batched_quantize( - tensor_bf16 - ) - torch.cuda.synchronize() - fp4_quant_time = (time.time() - start) / num_runs - - start = time.time() - for _ in range(num_runs): - tensor_fp4_dequant = FP4MXBlock16KVQuantizeUtil.batched_dequantize( - tensor_fp4, scale_factors - ) - torch.cuda.synchronize() - fp4_dequant_time = (time.time() - start) / num_runs - - fp4_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp4_dequant) - - return { - "fp8": { - "quant_time": fp8_quant_time, - "dequant_time": fp8_dequant_time, - **fp8_metrics, - }, - "fp4": { - "quant_time": fp4_quant_time, - "dequant_time": fp4_dequant_time, - **fp4_metrics, - }, - } - - -# default tensor shapes (m, n, k) -# [M, 1, 576]: DeepSeekR1-FP4 MLA -# [M, 8, 64]: gpt-oss-20b MHA -MNK_FACTORS = [ - (64, 1, 576), - (512, 1, 576), - (64, 8, 64), - (512, 8, 64), -] - - -@pytest.mark.parametrize("m,n,k", MNK_FACTORS) -def test_kvfp4_quant_dequant(m, n, k): - """Benchmark FP8 vs KVFP4 for predefined tensor shapes.""" - print(f"\n=== Running benchmark for tensor shape: [{m}, {n}, {k}] ===") - results = run_benchmark(m, n, k) - - print("FP8:", results["fp8"]) - print("FP4:", results["fp4"]) - - # Basic assertions to make sure metrics are reasonable - assert results["fp4"]["MSE"] < 0.1 - assert results["fp8"]["MSE"] < 0.1 - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py b/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py deleted file mode 100644 index db6f4eccb..000000000 --- a/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py +++ /dev/null @@ -1,186 +0,0 @@ -import unittest -from types import SimpleNamespace -from typing import Optional - -import requests - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_cuda_ci( - est_time=720, - stage="extra-a", - runner_config="2-gpu-large", - disabled="FIXME(kpham-sgl): temporary drop due to accuracies issue", -) - -MODEL_NAME = "26B-A4B" -TARGET_PATH = "google/gemma-4-26B-A4B-it" -ASSISTANT_PATH = "google/gemma-4-26B-A4B-it-assistant" -TENSOR_PARALLEL_SIZE = 2 - -TOPKS = (1, 3) -DRAFT_TOKENS_BY_TOPK = {1: 6, 3: 12} -GSM8K_NUM_EXAMPLES = 200 -GSM8K_NUM_THREADS = 128 -GSM8K_SCORE_MARGIN = 0.03 -SERVER_LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3 - -# Calibrated from deterministic-inference GSM8K runs (200 examples, 5-shot, -# greedy, triton, TP=2). With --enable-deterministic-inference the per-topk -# score is reproducible run-to-run (std=0 over N=20): topk=1 -> 0.445, -# topk=3 -> 0.440. -OBSERVED_GSM8K_SCORES = {1: 0.445, 3: 0.440} -GSM8K_SCORE_THRESHOLD = min(OBSERVED_GSM8K_SCORES.values()) - GSM8K_SCORE_MARGIN -ACCEPT_LENGTH_THRESHOLD = 1.5 - - -def get_server_info(base_url: str) -> dict: - response = requests.get(base_url + "/server_info", timeout=10) - response.raise_for_status() - return response.json() - - -def get_avg_spec_accept_length(base_url: str) -> Optional[float]: - try: - info = get_server_info(base_url) - except Exception: - return None - internal_states = info.get("internal_states") or [] - if not internal_states: - return None - value = internal_states[0].get("avg_spec_accept_length") - if value is None: - return None - return float(value) - - -class TestGemma4MTP26BA4B(CustomTestCase): - base_url = DEFAULT_URL_FOR_TEST - - @classmethod - def _common_server_args(cls) -> list[str]: - args = [ - "--attention-backend", - "triton", - "--dtype", - "bfloat16", - "--mem-fraction-static", - "0.55", - "--max-running-requests", - "16", - "--context-length", - "2048", - "--max-total-tokens", - "32768", - "--skip-server-warmup", - # Batch-invariant kernels make the GSM8K score reproducible - # run-to-run; without this the topk=3 score swings ~0.33-0.50. - "--enable-deterministic-inference", - ] - if TENSOR_PARALLEL_SIZE > 1: - args += ["--tp-size", str(TENSOR_PARALLEL_SIZE)] - return args - - @classmethod - def _server_args(cls, topk: int) -> list[str]: - return [ - "--speculative-algorithm", - "NEXTN", - "--speculative-draft-model-path", - ASSISTANT_PATH, - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - str(topk), - "--speculative-num-draft-tokens", - str(DRAFT_TOKENS_BY_TOPK[topk]), - ] + cls._common_server_args() - - @classmethod - def _gsm8k_args(cls) -> SimpleNamespace: - return SimpleNamespace( - base_url=cls.base_url, - model=TARGET_PATH, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=GSM8K_NUM_EXAMPLES, - num_threads=GSM8K_NUM_THREADS, - num_shots=5, - ) - - @staticmethod - def _stop_process(process) -> None: - try: - kill_process_tree(process.pid) - except Exception: - pass - - def _run_gsm8k_mtp(self, topk: int) -> None: - process = None - try: - process = popen_launch_server( - TARGET_PATH, - self.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=self._server_args(topk), - ) - requests.get(self.base_url + "/flush_cache", timeout=30) - - server_info = get_server_info(self.base_url) - self.assertEqual( - server_info.get("speculative_eagle_topk"), - topk, - f"{MODEL_NAME}: server did not start with topk={topk}", - ) - self.assertFalse( - bool(server_info.get("disable_cuda_graph")), - f"{MODEL_NAME}/topk{topk}: CUDA graph is disabled", - ) - - metrics = run_eval(self._gsm8k_args()) - mtp_score = float(metrics["score"]) - avg_accept = get_avg_spec_accept_length(self.base_url) - finally: - if process is not None: - self._stop_process(process) - - print( - f"[Gemma4 {MODEL_NAME} topk={topk}] " - f"score={mtp_score:.4f} threshold={GSM8K_SCORE_THRESHOLD:.4f} " - f"avg_spec_accept_length={avg_accept}" - ) - if is_in_ci(): - write_github_step_summary( - f"### Gemma4 {MODEL_NAME} MTP topk={topk}\n" - f"score={mtp_score:.4f}\n" - f"threshold={GSM8K_SCORE_THRESHOLD:.4f}\n" - f"avg_spec_accept_length={avg_accept}\n" - ) - - self.assertGreaterEqual(mtp_score, GSM8K_SCORE_THRESHOLD) - self.assertIsNotNone(avg_accept) - self.assertGreaterEqual( - avg_accept, - ACCEPT_LENGTH_THRESHOLD, - f"{MODEL_NAME}/topk{topk}: accept length too low", - ) - - def test_gsm8k_mtp(self) -> None: - for topk in TOPKS: - with self.subTest(topk=topk): - self._run_gsm8k_mtp(topk) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/layers/quantization/test_bf16_splitk_gemm.py b/test/registered/unit/layers/quantization/test_bf16_splitk_gemm.py index 381656d54..5e663cc0d 100644 --- a/test/registered/unit/layers/quantization/test_bf16_splitk_gemm.py +++ b/test/registered/unit/layers/quantization/test_bf16_splitk_gemm.py @@ -2,7 +2,6 @@ import pytest from sglang.srt.environ import envs from sglang.srt.layers.quantization.unquant import ( - _BF16_SPLITK_TUNED_TACTICS, Bf16GemmBackend, should_enable_bf16_splitk_gemm, use_bf16_splitk_gemm, @@ -12,11 +11,6 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") -@pytest.mark.parametrize("m,n,k", _BF16_SPLITK_TUNED_TACTICS) -def test_splitk_selects_tuned_oakhaven_shape(m: int, n: int, k: int): - assert use_bf16_splitk_gemm(m, n, k) - - @pytest.mark.parametrize("m", [0, 33, 64]) @pytest.mark.parametrize("n,k", [(256, 8192), (512, 8192), (2304, 8192), (2560, 8192)]) def test_splitk_keeps_large_m_on_existing_path(m: int, n: int, k: int): diff --git a/test/registered/utils/test_type_based_dispatcher.py b/test/registered/utils/test_type_based_dispatcher.py deleted file mode 100644 index 4f875d90e..000000000 --- a/test/registered/utils/test_type_based_dispatcher.py +++ /dev/null @@ -1,227 +0,0 @@ -# tests/benchmarks/test_type_dispatcher_e2e.py -""" -E2E test for TypeBasedDispatcher optimization. -Tests real-world scenarios with actual request types. -""" - -import timeit -import unittest - -from sglang.srt.managers.io_struct import SamplingParams -from sglang.test.ci.ci_register import register_amd_ci, register_cpu_ci -from sglang.utils import TypeBasedDispatcher - -register_amd_ci(est_time=10, suite="stage-b-test-1-gpu-small-amd") -register_cpu_ci(est_time=6, suite="stage-b-test-cpu-intel") - - -class TestTypeBasedDispatcher(unittest.TestCase): - """Unit tests for TypeBasedDispatcher e2e performance.""" - - def test_type_dispatcher_e2e_performance(self): - """End-to-end performance test with real request types""" - print("E2E Performance Test for TypeBasedDispatcher") - print("=" * 50) - - from sglang.srt.managers.io_struct import ( - AbortReq, - BatchTokenizedEmbeddingReqInput, - BatchTokenizedGenerateReqInput, - ClearHiCacheReqInput, - CloseSessionReqInput, - DestroyWeightsUpdateGroupReqInput, - ExpertDistributionReq, - FlushCacheReqInput, - FreezeGCReq, - GetInternalStateReq, - GetWeightsByNameReqInput, - InitWeightsSendGroupForRemoteInstanceReqInput, - InitWeightsUpdateGroupReqInput, - LoadLoRAAdapterReqInput, - OpenSessionReqInput, - ProfileReq, - ReleaseMemoryOccupationReqInput, - ResumeMemoryOccupationReqInput, - RpcReqInput, - SendWeightsToRemoteInstanceReqInput, - SetInternalStateReq, - SlowDownReqInput, - TokenizedEmbeddingReqInput, - TokenizedGenerateReqInput, - UnloadLoRAAdapterReqInput, - UpdateWeightFromDiskReqInput, - UpdateWeightsFromIPCReqInput, - UpdateWeightsFromTensorReqInput, - ) - - mapping = [ - (TokenizedGenerateReqInput, lambda req: "generate_handled"), - (TokenizedEmbeddingReqInput, lambda req: "embedding_handled"), - (BatchTokenizedGenerateReqInput, lambda req: "batch_generate_handled"), - ( - BatchTokenizedEmbeddingReqInput, - lambda req: "batch_generate_embedding_handled", - ), - (FlushCacheReqInput, lambda req: "flush_cache_handled"), - (ClearHiCacheReqInput, lambda req: "clear_hicache_handled"), - (AbortReq, lambda req: "abort_handled"), - (OpenSessionReqInput, lambda req: "open_session_handled"), - (CloseSessionReqInput, lambda req: "close_session_handled"), - ( - UpdateWeightFromDiskReqInput, - lambda req: "update_weights_from_disk_handled", - ), - ( - InitWeightsUpdateGroupReqInput, - lambda req: "init_weights_update_group_handled", - ), - ( - DestroyWeightsUpdateGroupReqInput, - lambda req: "destroy_weights_update_group_handled", - ), - ( - InitWeightsSendGroupForRemoteInstanceReqInput, - lambda req: "init_weights_send_group_for_remote_instance_handled", - ), - ( - SendWeightsToRemoteInstanceReqInput, - lambda req: "send_weights_to_remote_instance_handled", - ), - ( - UpdateWeightsFromTensorReqInput, - lambda req: "update_weights_from_tensor_handled", - ), - ( - UpdateWeightsFromIPCReqInput, - lambda req: "update_weights_from_ipc_handled", - ), - (GetWeightsByNameReqInput, lambda req: "get_weights_by_name_handled"), - ( - ReleaseMemoryOccupationReqInput, - lambda req: "release_memory_occupation_handled", - ), - ( - ResumeMemoryOccupationReqInput, - lambda req: "resume_memory_occupation_handled", - ), - (SlowDownReqInput, lambda req: "slow_down_handled"), - (ProfileReq, lambda req: "profile_handled"), - (FreezeGCReq, lambda req: "freeze_gc_handled"), - (GetInternalStateReq, lambda req: "get_internal_state_handled"), - (SetInternalStateReq, lambda req: "set_internal_state_handled"), - (RpcReqInput, lambda req: "rpc_request_handled"), - (ExpertDistributionReq, lambda req: "expert_distribution_handled"), - (LoadLoRAAdapterReqInput, lambda req: "load_lora_adapter_handled"), - (UnloadLoRAAdapterReqInput, lambda req: "unload_lora_adapter_handled"), - ] - - # Create requests that conforms to the real distribution - test_requests = [] - - test_requests.append( - TokenizedGenerateReqInput( - input_text="", - input_ids=[1, 2], - input_embeds=None, - mm_inputs=dict(), - token_type_ids=None, - sampling_params=SamplingParams(), - return_logprob=False, - logprob_start_len=0, - top_logprobs_num=0, - token_ids_logprob=[1, 2], - stream=False, - ) - ) - - test_requests.append( - TokenizedEmbeddingReqInput( - input_text="", - input_ids=[1, 2], - mm_inputs=dict(), - token_type_ids=[1, 2], - sampling_params=SamplingParams(), - ) - ) - - test_requests.append( - BatchTokenizedGenerateReqInput( - batch=[ - TokenizedGenerateReqInput( - input_text="", - input_ids=[1, 2], - input_embeds=None, - mm_inputs=dict(), - token_type_ids=None, - sampling_params=SamplingParams(), - return_logprob=False, - logprob_start_len=0, - top_logprobs_num=0, - token_ids_logprob=[1, 2], - stream=False, - ) - ] - ) - ) - test_requests.append( - BatchTokenizedEmbeddingReqInput( - batch=[ - TokenizedEmbeddingReqInput( - input_text="", - input_ids=[1, 2], - mm_inputs=dict(), - token_type_ids=[1, 2], - sampling_params=SamplingParams(), - ) - ] - ) - ) - - test_requests.append(FlushCacheReqInput()) - test_requests.append(ClearHiCacheReqInput()) - test_requests.append(AbortReq()) - test_requests.append(OpenSessionReqInput(capacity_of_str_len=0)) - test_requests.append(CloseSessionReqInput(session_id="")) - test_requests.append(UpdateWeightFromDiskReqInput(model_path="")) - test_requests.append( - InitWeightsUpdateGroupReqInput( - master_address="", - master_port=0, - rank_offset=0, - world_size=0, - group_name="", - ) - ) - test_requests.append(DestroyWeightsUpdateGroupReqInput()) - test_requests.append( - InitWeightsSendGroupForRemoteInstanceReqInput( - master_address="", ports="", group_name="", world_size=0, group_rank=0 - ) - ) - test_requests.append( - SendWeightsToRemoteInstanceReqInput(master_address="", ports="") - ) - test_requests.append( - UpdateWeightsFromTensorReqInput(serialized_named_tensors=[]) - ) - test_requests.append(GetWeightsByNameReqInput(name="")) - test_requests.append(ReleaseMemoryOccupationReqInput()) - test_requests.append(RpcReqInput(method="")) - - dispatcher = TypeBasedDispatcher(mapping) - - # test - time_taken = timeit.timeit( - lambda: [dispatcher(req) for req in test_requests], - number=100, # Average of 100 runs - ) - - print(f"Total requests: {len(test_requests)}") - print(f"Time taken: {time_taken:.4f}s") - print(f"Requests per second: {len(test_requests) * 100 / time_taken:.0f}") - - return time_taken - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/vlm/test_vision_openai_server_a.py b/test/registered/vlm/test_vision_openai_server_a.py index 531c97aa9..6910b7aed 100644 --- a/test/registered/vlm/test_vision_openai_server_a.py +++ b/test/registered/vlm/test_vision_openai_server_a.py @@ -229,8 +229,8 @@ class TestKimiVLServer(ImageOpenAITestMixin): "--mem-fraction-static=0.42", ] + @unittest.skip("model context length exceeded") def test_video_images_chat_completion(self): - # model context length exceeded pass