[CI] Graceful teardown for the PD and HiSparse server fixtures (#37485)

python/sglang/test/server_fixtures/disaggregation_fixture.py

@@ -19,6 +19,7 @@ from sglang.test.test_utils import (
     popen_launch_pd_server,
     popen_with_error_check,
     start_subprocess_fail_fast_watcher,
+    terminate_and_kill_process_tree,
 )
 from sglang.utils import wait_for_http_ready
 
@@ -225,10 +226,17 @@ class PDDisaggregationServerBase(CustomTestCase):
         os.environ.pop("MC_TCP_ENABLE_CONNECTION_POOL")
         if getattr(cls, "_mc_gid_index_set", False):
             os.environ.pop("MC_GID_INDEX", None)
-        for process in [cls.process_lb, cls.process_decode, cls.process_prefill]:
+        # The LB holds no device state, and popen_with_error_check only stays
+        # quiet for a SIGKILL rc, so hard-kill it rather than SIGTERM first.
+        if cls.process_lb:
+            try:
+                kill_process_tree(cls.process_lb.pid, wait_timeout=60)
+            except Exception as e:
+                print(f"Error killing process {cls.process_lb.pid}: {e}")
+        for process in [cls.process_decode, cls.process_prefill]:
             if process:
                 try:
-                    kill_process_tree(process.pid, wait_timeout=60)
+                    terminate_and_kill_process_tree(process, wait_timeout=60)
                 except Exception as e:
                     print(f"Error killing process {process.pid}: {e}")
 

test/registered/amd/accuracy/mi30x/test_glm51_hisparse_eval_mi30x.py

@@ -3,13 +3,13 @@
 import unittest
 from types import SimpleNamespace
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     is_in_ci,
     popen_launch_server,
+    terminate_and_kill_process_tree,
     write_github_step_summary,
 )
 
@@ -62,7 +62,7 @@ class TestGLM51HiSparseEvalAMD(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         if hasattr(cls, "process"):
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k_accuracy(self):
         args = SimpleNamespace(

test/registered/amd/accuracy/mi35x/test_glm51_hisparse_eval_mi35x.py

@@ -3,13 +3,13 @@
 import unittest
 from types import SimpleNamespace
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     is_in_ci,
     popen_launch_server,
+    terminate_and_kill_process_tree,
     write_github_step_summary,
 )
 
@@ -62,7 +62,7 @@ class TestGLM51HiSparseEvalMI35x(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         if hasattr(cls, "process"):
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k_accuracy(self):
         args = SimpleNamespace(

test/registered/models_e2e/test_dsa_glm52_hisparse.py

@@ -1,11 +1,10 @@
-import subprocess
 import time
 import unittest
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
 from sglang.test.server_fixtures.default_fixture import DefaultServerBase
+from sglang.test.test_utils import terminate_and_kill_process_tree
 
 register_cuda_ci(est_time=720, stage="extra-b", runner_config="8-gpu-h200")
 
@@ -54,14 +53,10 @@ class TestGLM52HiSparse(DefaultServerBase, GSM8KMixin):
 
     @classmethod
     def tearDownClass(cls):
-        # HiSparse's large pinned host buffer stalls an external SIGKILL teardown
+        # HiSparse's pinned host buffer needs longer than the base class's 60s.
-        # (kernel unpin). Drive the server's own graceful shutdown so each rank
+        terminate_and_kill_process_tree(
-        # unregisters in userspace; hard-kill as a fallback.
+            cls.process, terminate_timeout=90, wait_timeout=60
-        cls.process.terminate()
+        )
-        try:
-            cls.process.wait(timeout=90)
-        except subprocess.TimeoutExpired:
-            kill_process_tree(cls.process.pid, wait_timeout=60)
         time.sleep(2)
 
 

test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py

@@ -26,7 +26,6 @@ import unittest
 from types import SimpleNamespace
 from urllib.parse import urlparse
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
 from sglang.test.server_fixtures.default_fixture import (
@@ -36,6 +35,7 @@ from sglang.test.server_fixtures.default_fixture import (
 from sglang.test.test_utils import (
     DEFAULT_HYBRID_MAMBA_MODEL_NAME_FOR_TEST,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-h100")
@@ -115,7 +115,7 @@ class TestUnifiedRadixTreeInt8MambaCheckpointE2E(TestInt8MambaCheckpointE2E):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         time.sleep(2)
 
 

test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dcp.py

@@ -7,7 +7,6 @@ hits return another rank's KV. The KL cases catch that as a large divergence.
 Blackwell-only: the MLA DCP decode path needs ``tokenspeed_mla`` (SM100/12x).
 """
 
-import subprocess
 import unittest
 
 from sglang.test.ci.ci_register import register_cuda_ci
@@ -99,11 +98,6 @@ class TestUnifiedKimiLinearDcpHiCache(UnifiedRadixTreeTestMixin, CustomTestCase)
 
     @classmethod
     def tearDownClass(cls):
-        cls.process.terminate()
-        try:
-            cls.process.wait(timeout=60)
-        except subprocess.TimeoutExpired:
-            pass
         terminate_and_kill_process_tree(cls.process, wait_timeout=60)