[Fix] Shut hicache test servers down gracefully before SIGKILL (#37194)

test/registered/amd/test_deepseek_r1_hicache_mi35x.py

@@ -21,7 +21,6 @@ import tempfile
 import unittest
 from types import SimpleNamespace
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
@@ -29,6 +28,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
     is_in_ci,
     popen_launch_server,
+    terminate_and_kill_process_tree,
     write_github_step_summary,
 )
 
@@ -129,7 +129,7 @@ class TestDeepSeekR1HiCacheMI35x(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
         if getattr(cls, "l3_storage_dir", None):
             shutil.rmtree(cls.l3_storage_dir, ignore_errors=True)
 

test/registered/disaggregation/test_disaggregation_decode_offload.py

@@ -3,7 +3,6 @@ import shutil
 import unittest
 from types import SimpleNamespace
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
@@ -13,6 +12,7 @@ from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     popen_launch_pd_server,
+    terminate_and_kill_process_tree,
 )
 
 # Registering the test for CUDA CI with appropriate parameters
@@ -150,12 +150,9 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
 
         time.sleep(10)
 
-        kill_process_tree(self.process_prefill.pid)
-        kill_process_tree(self.process_decode.pid)
-        kill_process_tree(self.process_lb.pid)
-        self.process_prefill.wait()
-        self.process_decode.wait()
-        self.process_lb.wait()
+        terminate_and_kill_process_tree(self.process_prefill)
+        terminate_and_kill_process_tree(self.process_decode)
+        terminate_and_kill_process_tree(self.process_lb)
 
         self.start_prefill()
         self.start_decode()

test/registered/hicache/test_hicache_storage.py

@@ -3,11 +3,10 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 register_cuda_ci(est_time=99, stage="base-b", runner_config="1-gpu-small")
 register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
 
-import subprocess
 import time
 import unittest
 
-from sglang.srt.utils import is_hip, kill_process_tree
+from sglang.srt.utils import is_hip
 from sglang.test.kits.eval_accuracy_kit import MMLUMixin
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -15,6 +14,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 _is_hip = is_hip()
@@ -48,15 +48,7 @@ class TestHiCache(CustomTestCase, MMLUMixin):
 
     @classmethod
     def tearDownClass(cls):
-        # Graceful stop first so the server unregisters its large pinned host KV
-        # pool in userspace; a bare SIGKILL leaves the kernel to unpin it during
-        # reclaim, holding GPU memory long enough to fail the next test.
-        cls.process.terminate()
-        try:
-            cls.process.wait(timeout=60)
-        except subprocess.TimeoutExpired:
-            pass
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process)
         time.sleep(5)
 
 

test/registered/hicache/test_hicache_storage_file_backend.py

@@ -17,7 +17,6 @@ from urllib.parse import urlparse
 import requests
 
 from sglang.benchmark.utils import get_tokenizer
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
@@ -28,6 +27,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
     is_in_ci,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 from sglang.utils import wait_for_http_ready
 
@@ -63,7 +63,7 @@ class HiCacheStorageBaseMixin:
     def tearDownClass(cls):
         """Clean up test environment"""
         if hasattr(cls, "process") and cls.process:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
 
         import shutil
 

test/registered/hicache/test_hicache_storage_runtime_attach_detach.py

@@ -16,7 +16,6 @@ import time
 import unittest
 from urllib import error, request
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -25,6 +24,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
     find_available_port,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 from sglang.utils import wait_for_http_ready
 
@@ -222,7 +222,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
             )
             self.assertEqual(code_detach_no_admin, 400)
         finally:
-            kill_process_tree(process1.pid)
+            terminate_and_kill_process_tree(process1)
             time.sleep(2)
 
         self._check_attach_detach_lifecycle()
@@ -367,7 +367,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
             )
             self.assertEqual(code_detach2, 200, f"{code_detach2} - {body_detach2}")
         finally:
-            kill_process_tree(process2.pid)
+            terminate_and_kill_process_tree(process2)
             time.sleep(2)
 
 

test/registered/hicache/test_hicache_variants.py

@@ -10,7 +10,7 @@ Tests HiCache with different configurations: standard, MLA, EAGLE, and page size
 import unittest
 
 from sglang.benchmark.utils import get_tokenizer
-from sglang.srt.utils import is_hip, kill_process_tree
+from sglang.srt.utils import is_hip
 from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE3,
@@ -21,6 +21,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 _is_hip = is_hip()
@@ -50,7 +51,7 @@ class HiCacheBaseServer(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process)
 
 
 class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):

test/registered/hicache/test_pp_with_hicache.py

@@ -16,13 +16,13 @@ from urllib.parse import urlparse
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     find_available_port,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 
@@ -74,7 +74,7 @@ class TestPPWithHiCache(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         if hasattr(cls, "process"):
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
         cls._stop_mooncake_services()
 
     @classmethod

test/registered/hicache/test_qwen35_hicache.py

@@ -9,7 +9,6 @@ import zmq
 from msgspec.msgpack import Decoder
 
 from sglang.srt.disaggregation.kv_events import BlockStored, KVEventBatch
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 
 # This eval harness applies the chat_template, which is critical for qwen3.5
@@ -20,6 +19,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=540, stage="extra-b", runner_config="4-gpu-h100")
@@ -93,7 +93,7 @@ class TestQwen35WithHiCache(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process)
         shutil.rmtree(cls.storage_dir, ignore_errors=True)
 
     def _run_gsm8k(self):

test/registered/models_e2e/test_inkling_small_nvfp4.py

@@ -17,7 +17,6 @@ import unittest
 from types import SimpleNamespace
 from urllib.parse import urlparse
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 
 # Aliased for the same reason as the single-turn helpers above.
@@ -46,6 +45,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=2000, stage="extra-b", runner_config="4-gpu-b200")
@@ -118,7 +118,7 @@ class TestInklingSmallNvfp4(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
 
     def test_gsm8k(self):
         """Answer quality on the real checkpoint: guards the modelopt_fp4 weight
@@ -208,7 +208,7 @@ class TestInklingSmallNvfp4DsparkDeterministic(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
 
     def _run(self, helper, **kwargs):
         helper(
@@ -323,7 +323,7 @@ class TestInklingSmallNvfp4HiCacheDeterministic(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process)
 
     def test_multiturn_decode_cache_hit_over_hicache(self):
         """Nine interleaved branches, three turns, decode hits served through the

test/registered/models_e2e/test_kimi_k3_b300.py

@@ -6,7 +6,6 @@ their GSM8K accuracy gates.
 
 import unittest
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
 from sglang.test.test_utils import (
@@ -14,6 +13,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
     _wait_for_gpu_idle_in_ci,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=1200, stage="base-c", runner_config="8-gpu-b300")
@@ -31,7 +31,7 @@ GPU_IDLE_TIMEOUT = 120
 
 def _stop_server(process):
     if process:
-        kill_process_tree(process.pid)
+        terminate_and_kill_process_tree(process)
         _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT)
 
 

test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py

@@ -2,7 +2,6 @@ import unittest
 from types import SimpleNamespace
 from urllib.parse import urlparse
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
@@ -11,6 +10,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3")
@@ -72,7 +72,7 @@ class TestAscendMhaHicache(CustomTestCase):
                             TEST_MODEL_MATRIX[model]["accuracy"],
                         )
                 finally:
-                    kill_process_tree(process.pid)
+                    terminate_and_kill_process_tree(process)
 
 
 if __name__ == "__main__":

test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py

@@ -2,7 +2,6 @@ import unittest
 from types import SimpleNamespace
 from urllib.parse import urlparse
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
@@ -11,6 +10,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
@@ -79,7 +79,7 @@ class TestAscendMlaHicache(CustomTestCase):
                             TEST_MODEL_MATRIX[model]["accuracy"],
                         )
                 finally:
-                    kill_process_tree(process.pid)
+                    terminate_and_kill_process_tree(process)
 
 
 if __name__ == "__main__":

test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py

@@ -2,7 +2,6 @@ import unittest
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import (
@@ -10,6 +9,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
@@ -51,7 +51,7 @@ class TestNPUHierarchicalCache(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process)
 
     def test_hierarchical_cache_reused_long_identical(self):
         """Long identical texts should reuse HierarchicalCache"""