From 3a6ed5599995bcf83650e2e336863b15b6658695 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 30 Aug 2026 22:45:40 -0700 Subject: [PATCH] [Fix] Shut hicache test servers down gracefully before SIGKILL (#37194) --- .../amd/test_deepseek_r1_hicache_mi35x.py | 4 ++-- .../test_disaggregation_decode_offload.py | 11 ++++------- test/registered/hicache/test_hicache_storage.py | 14 +++----------- .../hicache/test_hicache_storage_file_backend.py | 4 ++-- .../test_hicache_storage_runtime_attach_detach.py | 6 +++--- test/registered/hicache/test_hicache_variants.py | 5 +++-- test/registered/hicache/test_pp_with_hicache.py | 4 ++-- test/registered/hicache/test_qwen35_hicache.py | 4 ++-- .../models_e2e/test_inkling_small_nvfp4.py | 8 ++++---- test/registered/models_e2e/test_kimi_k3_b300.py | 4 ++-- .../basic_function/HiCache/test_npu_hicache_mha.py | 4 ++-- .../basic_function/HiCache/test_npu_hicache_mla.py | 4 ++-- .../HiCache/test_npu_hierarchical_cache.py | 4 ++-- 13 files changed, 33 insertions(+), 43 deletions(-) diff --git a/test/registered/amd/test_deepseek_r1_hicache_mi35x.py b/test/registered/amd/test_deepseek_r1_hicache_mi35x.py index 4010536be..f52011bf7 100644 --- a/test/registered/amd/test_deepseek_r1_hicache_mi35x.py +++ b/test/registered/amd/test_deepseek_r1_hicache_mi35x.py @@ -21,7 +21,6 @@ import tempfile import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( @@ -29,6 +28,7 @@ from sglang.test.test_utils import ( CustomTestCase, is_in_ci, popen_launch_server, + terminate_and_kill_process_tree, write_github_step_summary, ) @@ -129,7 +129,7 @@ class TestDeepSeekR1HiCacheMI35x(CustomTestCase): @classmethod def tearDownClass(cls): if getattr(cls, "process", None) is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) if getattr(cls, "l3_storage_dir", None): shutil.rmtree(cls.l3_storage_dir, ignore_errors=True) diff --git a/test/registered/disaggregation/test_disaggregation_decode_offload.py b/test/registered/disaggregation/test_disaggregation_decode_offload.py index 76221105b..05f4eb1f7 100644 --- a/test/registered/disaggregation/test_disaggregation_decode_offload.py +++ b/test/registered/disaggregation/test_disaggregation_decode_offload.py @@ -3,7 +3,6 @@ import shutil import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( @@ -13,6 +12,7 @@ from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, popen_launch_pd_server, + terminate_and_kill_process_tree, ) # Registering the test for CUDA CI with appropriate parameters @@ -150,12 +150,9 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): time.sleep(10) - kill_process_tree(self.process_prefill.pid) - kill_process_tree(self.process_decode.pid) - kill_process_tree(self.process_lb.pid) - self.process_prefill.wait() - self.process_decode.wait() - self.process_lb.wait() + terminate_and_kill_process_tree(self.process_prefill) + terminate_and_kill_process_tree(self.process_decode) + terminate_and_kill_process_tree(self.process_lb) self.start_prefill() self.start_decode() diff --git a/test/registered/hicache/test_hicache_storage.py b/test/registered/hicache/test_hicache_storage.py index e684130e8..5f5e622cc 100644 --- a/test/registered/hicache/test_hicache_storage.py +++ b/test/registered/hicache/test_hicache_storage.py @@ -3,11 +3,10 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=99, stage="base-b", runner_config="1-gpu-small") register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd") -import subprocess import time import unittest -from sglang.srt.utils import is_hip, kill_process_tree +from sglang.srt.utils import is_hip from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -15,6 +14,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) _is_hip = is_hip() @@ -48,15 +48,7 @@ class TestHiCache(CustomTestCase, MMLUMixin): @classmethod def tearDownClass(cls): - # Graceful stop first so the server unregisters its large pinned host KV - # pool in userspace; a bare SIGKILL leaves the kernel to unpin it during - # reclaim, holding GPU memory long enough to fail the next test. - cls.process.terminate() - try: - cls.process.wait(timeout=60) - except subprocess.TimeoutExpired: - pass - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) time.sleep(5) diff --git a/test/registered/hicache/test_hicache_storage_file_backend.py b/test/registered/hicache/test_hicache_storage_file_backend.py index 0fd459c3b..37860a1cb 100644 --- a/test/registered/hicache/test_hicache_storage_file_backend.py +++ b/test/registered/hicache/test_hicache_storage_file_backend.py @@ -17,7 +17,6 @@ from urllib.parse import urlparse import requests from sglang.benchmark.utils import get_tokenizer -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( @@ -28,6 +27,7 @@ from sglang.test.test_utils import ( CustomTestCase, is_in_ci, popen_launch_server, + terminate_and_kill_process_tree, ) from sglang.utils import wait_for_http_ready @@ -63,7 +63,7 @@ class HiCacheStorageBaseMixin: def tearDownClass(cls): """Clean up test environment""" if hasattr(cls, "process") and cls.process: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) import shutil diff --git a/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py b/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py index 2794ba091..6e8d74043 100644 --- a/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py +++ b/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py @@ -16,7 +16,6 @@ import time import unittest from urllib import error, request -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -25,6 +24,7 @@ from sglang.test.test_utils import ( CustomTestCase, find_available_port, popen_launch_server, + terminate_and_kill_process_tree, ) from sglang.utils import wait_for_http_ready @@ -222,7 +222,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase): ) self.assertEqual(code_detach_no_admin, 400) finally: - kill_process_tree(process1.pid) + terminate_and_kill_process_tree(process1) time.sleep(2) self._check_attach_detach_lifecycle() @@ -367,7 +367,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase): ) self.assertEqual(code_detach2, 200, f"{code_detach2} - {body_detach2}") finally: - kill_process_tree(process2.pid) + terminate_and_kill_process_tree(process2) time.sleep(2) diff --git a/test/registered/hicache/test_hicache_variants.py b/test/registered/hicache/test_hicache_variants.py index 1f130dd2d..5dac5764f 100644 --- a/test/registered/hicache/test_hicache_variants.py +++ b/test/registered/hicache/test_hicache_variants.py @@ -10,7 +10,7 @@ Tests HiCache with different configurations: standard, MLA, EAGLE, and page size import unittest from sglang.benchmark.utils import get_tokenizer -from sglang.srt.utils import is_hip, kill_process_tree +from sglang.srt.utils import is_hip from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, @@ -21,6 +21,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) _is_hip = is_hip() @@ -50,7 +51,7 @@ class HiCacheBaseServer(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin): diff --git a/test/registered/hicache/test_pp_with_hicache.py b/test/registered/hicache/test_pp_with_hicache.py index 57226b4dc..3c69b3759 100644 --- a/test/registered/hicache/test_pp_with_hicache.py +++ b/test/registered/hicache/test_pp_with_hicache.py @@ -16,13 +16,13 @@ from urllib.parse import urlparse import requests -from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, find_available_port, popen_launch_server, + terminate_and_kill_process_tree, ) @@ -74,7 +74,7 @@ class TestPPWithHiCache(unittest.TestCase): @classmethod def tearDownClass(cls): if hasattr(cls, "process"): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) cls._stop_mooncake_services() @classmethod diff --git a/test/registered/hicache/test_qwen35_hicache.py b/test/registered/hicache/test_qwen35_hicache.py index 7b271917d..1a912af5e 100644 --- a/test/registered/hicache/test_qwen35_hicache.py +++ b/test/registered/hicache/test_qwen35_hicache.py @@ -9,7 +9,6 @@ import zmq from msgspec.msgpack import Decoder from sglang.srt.disaggregation.kv_events import BlockStored, KVEventBatch -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci # This eval harness applies the chat_template, which is critical for qwen3.5 @@ -20,6 +19,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=540, stage="extra-b", runner_config="4-gpu-h100") @@ -93,7 +93,7 @@ class TestQwen35WithHiCache(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) shutil.rmtree(cls.storage_dir, ignore_errors=True) def _run_gsm8k(self): diff --git a/test/registered/models_e2e/test_inkling_small_nvfp4.py b/test/registered/models_e2e/test_inkling_small_nvfp4.py index 95a091b61..6e02f6729 100644 --- a/test/registered/models_e2e/test_inkling_small_nvfp4.py +++ b/test/registered/models_e2e/test_inkling_small_nvfp4.py @@ -17,7 +17,6 @@ import unittest from types import SimpleNamespace from urllib.parse import urlparse -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci # Aliased for the same reason as the single-turn helpers above. @@ -46,6 +45,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=2000, stage="extra-b", runner_config="4-gpu-b200") @@ -118,7 +118,7 @@ class TestInklingSmallNvfp4(CustomTestCase): @classmethod def tearDownClass(cls): if getattr(cls, "process", None) is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) def test_gsm8k(self): """Answer quality on the real checkpoint: guards the modelopt_fp4 weight @@ -208,7 +208,7 @@ class TestInklingSmallNvfp4DsparkDeterministic(CustomTestCase): @classmethod def tearDownClass(cls): if getattr(cls, "process", None) is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) def _run(self, helper, **kwargs): helper( @@ -323,7 +323,7 @@ class TestInklingSmallNvfp4HiCacheDeterministic(CustomTestCase): @classmethod def tearDownClass(cls): if getattr(cls, "process", None) is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) def test_multiturn_decode_cache_hit_over_hicache(self): """Nine interleaved branches, three turns, decode hits served through the diff --git a/test/registered/models_e2e/test_kimi_k3_b300.py b/test/registered/models_e2e/test_kimi_k3_b300.py index 7fab627e2..5eb20b772 100644 --- a/test/registered/models_e2e/test_kimi_k3_b300.py +++ b/test/registered/models_e2e/test_kimi_k3_b300.py @@ -6,7 +6,6 @@ their GSM8K accuracy gates. import unittest -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.test_utils import ( @@ -14,6 +13,7 @@ from sglang.test.test_utils import ( CustomTestCase, _wait_for_gpu_idle_in_ci, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=1200, stage="base-c", runner_config="8-gpu-b300") @@ -31,7 +31,7 @@ GPU_IDLE_TIMEOUT = 120 def _stop_server(process): if process: - kill_process_tree(process.pid) + terminate_and_kill_process_tree(process) _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT) diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py index 99b843ed2..4647549ff 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py @@ -2,7 +2,6 @@ import unittest from types import SimpleNamespace from urllib.parse import urlparse -from sglang.srt.utils import kill_process_tree from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k @@ -11,6 +10,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") @@ -72,7 +72,7 @@ class TestAscendMhaHicache(CustomTestCase): TEST_MODEL_MATRIX[model]["accuracy"], ) finally: - kill_process_tree(process.pid) + terminate_and_kill_process_tree(process) if __name__ == "__main__": diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py index 496a38373..08214ff48 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py @@ -2,7 +2,6 @@ import unittest from types import SimpleNamespace from urllib.parse import urlparse -from sglang.srt.utils import kill_process_tree from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k @@ -11,6 +10,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") @@ -79,7 +79,7 @@ class TestAscendMlaHicache(CustomTestCase): TEST_MODEL_MATRIX[model]["accuracy"], ) finally: - kill_process_tree(process.pid) + terminate_and_kill_process_tree(process) if __name__ == "__main__": diff --git a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py index 011240353..9210fa1d9 100644 --- a/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py +++ b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py @@ -2,7 +2,6 @@ import unittest import requests -from sglang.srt.utils import kill_process_tree from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import ( @@ -10,6 +9,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) @@ -51,7 +51,7 @@ class TestNPUHierarchicalCache(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process) def test_hierarchical_cache_reused_long_identical(self): """Long identical texts should reuse HierarchicalCache"""