From f5c225eeba3c6f027e5e7691784249d6a667026f Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Wed, 25 Mar 2026 00:04:59 -0700 Subject: [PATCH] [CI] Fix TestQwen35WithHiCache (#21371) --- .../4-gpu-models/test_qwen35_hicache.py | 127 ++++++++++++++++++ .../4-gpu-models/test_qwen35_models.py | 104 +------------- 2 files changed, 128 insertions(+), 103 deletions(-) create mode 100644 test/registered/4-gpu-models/test_qwen35_hicache.py diff --git a/test/registered/4-gpu-models/test_qwen35_hicache.py b/test/registered/4-gpu-models/test_qwen35_hicache.py new file mode 100644 index 000000000..90011b401 --- /dev/null +++ b/test/registered/4-gpu-models/test_qwen35_hicache.py @@ -0,0 +1,127 @@ +import shutil +import tempfile +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci + +# This eval harness applies the chat_template, which is critical for qwen3.5 +# to get good accuracy on gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=600, suite="stage-c-test-4-gpu-h100") + +QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B" +ACC_THRESHOLDS = {QWEN35_27B_MODEL: {"gsm8k": 0.8}} + + +class TestQwen35WithHiCache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = QWEN35_27B_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.storage_dir = tempfile.mkdtemp(prefix="qwen35-hicache-") + env = { + "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.storage_dir, + } + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + env=env, + other_args=[ + "--tp-size", + "4", + "--max-mamba-cache-size", + "500", + "--max-total-tokens", + "120000", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + "128", + "--mamba-ssm-dtype", + "bfloat16", + "--max-running-requests", + "128", + "--reasoning-parser", + "qwen3", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + "--hicache-mem-layout", + "page_first_direct", + "--enable-hierarchical-cache", + "--hicache-ratio", + "2", + "--hicache-size", + "0", + "--hicache-write-policy", + "write_through", + "--hicache-storage-backend", + "file", + "--hicache-storage-prefetch-policy", + "wait_complete", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + shutil.rmtree(cls.storage_dir, ignore_errors=True) + + def _run_gsm8k(self): + args = SimpleNamespace( + model=self.model, + eval_name="gsm8k", + num_shots=5, + num_examples=100, + max_tokens=16000, + num_threads=50, + repeat=1, + temperature=0.6, + top_p=0.95, + top_k=20, + base_url=self.base_url, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + return run_eval(args) + + def test_gsm8k(self): + first_metrics = self._run_gsm8k() + print(f"first_metrics={first_metrics}") + self.assertGreaterEqual( + first_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"] + ) + + print(f"flush cache") + time.sleep(2) + requests.post(f"{self.base_url}/flush_cache", timeout=10) + + second_metrics = self._run_gsm8k() + print(f"second_metrics={second_metrics}") + self.assertGreaterEqual( + second_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"] + ) + self.assertLessEqual( + abs(second_metrics["score"] - first_metrics["score"]), + 0.05, + f"HiCache prefetch accuracy drift too large: " + f"first={first_metrics['score']}, second={second_metrics['score']}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/4-gpu-models/test_qwen35_models.py b/test/registered/4-gpu-models/test_qwen35_models.py index cfd1626f0..f088c8242 100644 --- a/test/registered/4-gpu-models/test_qwen35_models.py +++ b/test/registered/4-gpu-models/test_qwen35_models.py @@ -1,6 +1,3 @@ -import shutil -import tempfile -import time import unittest from types import SimpleNamespace @@ -26,8 +23,7 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=1400, suite="stage-c-test-4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" -QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B" -ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}, QWEN35_27B_MODEL: {"gsm8k": 0.8}} +ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} class TestQwen35FP4(unittest.TestCase): @@ -238,103 +234,5 @@ class TestQwen35FP4MTPV2(CustomTestCase): self.assertGreater(avg_spec_accept_length, 3.3) -class TestQwen35WithHiCache(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = QWEN35_27B_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.storage_dir = tempfile.mkdtemp(prefix="qwen35-hicache-") - env = { - "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.storage_dir, - } - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - env=env, - other_args=[ - "--tp-size", - "4", - "--max-mamba-cache-size", - "500", - "--max-total-tokens", - "120000", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - "128", - "--mamba-ssm-dtype", - "bfloat16", - "--max-running-requests", - "128", - "--reasoning-parser", - "qwen3", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - "--hicache-mem-layout", - "page_first_direct", - "--enable-hierarchical-cache", - "--hicache-ratio", - "2", - "--hicache-size", - "0", - "--hicache-write-policy", - "write_through", - "--hicache-storage-backend", - "file", - "--hicache-storage-prefetch-policy", - "wait_complete", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - shutil.rmtree(cls.storage_dir, ignore_errors=True) - - def _run_gsm8k(self): - args = SimpleNamespace( - model=self.model, - eval_name="gsm8k", - num_shots=5, - num_examples=100, - max_tokens=4000, - num_threads=50, - repeat=1, - temperature=0.6, - top_p=0.95, - top_k=20, - base_url=self.base_url, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - return run_eval(args) - - def test_gsm8k(self): - first_metrics = self._run_gsm8k() - print(f"first_metrics={first_metrics}") - self.assertGreaterEqual( - first_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"] - ) - - print(f"flush cache") - time.sleep(2) - requests.post(f"{self.base_url}/flush_cache", timeout=10) - - second_metrics = self._run_gsm8k() - print(f"second_metrics={second_metrics}") - self.assertGreaterEqual( - second_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"] - ) - self.assertLessEqual( - abs(second_metrics["score"] - first_metrics["score"]), - 0.05, - f"HiCache prefetch accuracy drift too large: " - f"first={first_metrics['score']}, second={second_metrics['score']}", - ) - - if __name__ == "__main__": unittest.main()