From db1eb486518aa149058403f58eaebab04a140818 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:22:18 -0700 Subject: [PATCH] [CI] Graceful teardown for the PD and HiSparse server fixtures (#37485) --- .../server_fixtures/disaggregation_fixture.py | 12 ++++++++++-- .../mi30x/test_glm51_hisparse_eval_mi30x.py | 4 ++-- .../mi35x/test_glm51_hisparse_eval_mi35x.py | 4 ++-- .../models_e2e/test_dsa_glm52_hisparse.py | 15 +++++---------- .../radix_cache/test_int8_mamba_checkpoint_e2e.py | 4 ++-- .../test_unified_radix_cache_kl_dcp.py | 6 ------ 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/python/sglang/test/server_fixtures/disaggregation_fixture.py b/python/sglang/test/server_fixtures/disaggregation_fixture.py index cdf99b246..0aba39f7e 100644 --- a/python/sglang/test/server_fixtures/disaggregation_fixture.py +++ b/python/sglang/test/server_fixtures/disaggregation_fixture.py @@ -19,6 +19,7 @@ from sglang.test.test_utils import ( popen_launch_pd_server, popen_with_error_check, start_subprocess_fail_fast_watcher, + terminate_and_kill_process_tree, ) from sglang.utils import wait_for_http_ready @@ -225,10 +226,17 @@ class PDDisaggregationServerBase(CustomTestCase): os.environ.pop("MC_TCP_ENABLE_CONNECTION_POOL") if getattr(cls, "_mc_gid_index_set", False): os.environ.pop("MC_GID_INDEX", None) - for process in [cls.process_lb, cls.process_decode, cls.process_prefill]: + # The LB holds no device state, and popen_with_error_check only stays + # quiet for a SIGKILL rc, so hard-kill it rather than SIGTERM first. + if cls.process_lb: + try: + kill_process_tree(cls.process_lb.pid, wait_timeout=60) + except Exception as e: + print(f"Error killing process {cls.process_lb.pid}: {e}") + for process in [cls.process_decode, cls.process_prefill]: if process: try: - kill_process_tree(process.pid, wait_timeout=60) + terminate_and_kill_process_tree(process, wait_timeout=60) except Exception as e: print(f"Error killing process {process.pid}: {e}") diff --git a/test/registered/amd/accuracy/mi30x/test_glm51_hisparse_eval_mi30x.py b/test/registered/amd/accuracy/mi30x/test_glm51_hisparse_eval_mi30x.py index 058b0d650..cca2da359 100644 --- a/test/registered/amd/accuracy/mi30x/test_glm51_hisparse_eval_mi30x.py +++ b/test/registered/amd/accuracy/mi30x/test_glm51_hisparse_eval_mi30x.py @@ -3,13 +3,13 @@ import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, is_in_ci, popen_launch_server, + terminate_and_kill_process_tree, write_github_step_summary, ) @@ -62,7 +62,7 @@ class TestGLM51HiSparseEvalAMD(unittest.TestCase): @classmethod def tearDownClass(cls): if hasattr(cls, "process"): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_gsm8k_accuracy(self): args = SimpleNamespace( diff --git a/test/registered/amd/accuracy/mi35x/test_glm51_hisparse_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_glm51_hisparse_eval_mi35x.py index a50d56d1a..9dc8a0740 100644 --- a/test/registered/amd/accuracy/mi35x/test_glm51_hisparse_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_glm51_hisparse_eval_mi35x.py @@ -3,13 +3,13 @@ import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, is_in_ci, popen_launch_server, + terminate_and_kill_process_tree, write_github_step_summary, ) @@ -62,7 +62,7 @@ class TestGLM51HiSparseEvalMI35x(unittest.TestCase): @classmethod def tearDownClass(cls): if hasattr(cls, "process"): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_gsm8k_accuracy(self): args = SimpleNamespace( diff --git a/test/registered/models_e2e/test_dsa_glm52_hisparse.py b/test/registered/models_e2e/test_dsa_glm52_hisparse.py index da3089d91..231e6155f 100644 --- a/test/registered/models_e2e/test_dsa_glm52_hisparse.py +++ b/test/registered/models_e2e/test_dsa_glm52_hisparse.py @@ -1,11 +1,10 @@ -import subprocess import time import unittest -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_utils import terminate_and_kill_process_tree register_cuda_ci(est_time=720, stage="extra-b", runner_config="8-gpu-h200") @@ -54,14 +53,10 @@ class TestGLM52HiSparse(DefaultServerBase, GSM8KMixin): @classmethod def tearDownClass(cls): - # HiSparse's large pinned host buffer stalls an external SIGKILL teardown - # (kernel unpin). Drive the server's own graceful shutdown so each rank - # unregisters in userspace; hard-kill as a fallback. - cls.process.terminate() - try: - cls.process.wait(timeout=90) - except subprocess.TimeoutExpired: - kill_process_tree(cls.process.pid, wait_timeout=60) + # HiSparse's pinned host buffer needs longer than the base class's 60s. + terminate_and_kill_process_tree( + cls.process, terminate_timeout=90, wait_timeout=60 + ) time.sleep(2) diff --git a/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py b/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py index ea1a6a973..8e89e9dc0 100644 --- a/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py +++ b/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py @@ -26,7 +26,6 @@ import unittest from types import SimpleNamespace from urllib.parse import urlparse -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin from sglang.test.server_fixtures.default_fixture import ( @@ -36,6 +35,7 @@ from sglang.test.server_fixtures.default_fixture import ( from sglang.test.test_utils import ( DEFAULT_HYBRID_MAMBA_MODEL_NAME_FOR_TEST, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-h100") @@ -115,7 +115,7 @@ class TestUnifiedRadixTreeInt8MambaCheckpointE2E(TestInt8MambaCheckpointE2E): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid, wait_timeout=60) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) time.sleep(2) diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dcp.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dcp.py index ba81eaa4b..b048f5290 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dcp.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dcp.py @@ -7,7 +7,6 @@ hits return another rank's KV. The KL cases catch that as a large divergence. Blackwell-only: the MLA DCP decode path needs ``tokenspeed_mla`` (SM100/12x). """ -import subprocess import unittest from sglang.test.ci.ci_register import register_cuda_ci @@ -99,11 +98,6 @@ class TestUnifiedKimiLinearDcpHiCache(UnifiedRadixTreeTestMixin, CustomTestCase) @classmethod def tearDownClass(cls): - cls.process.terminate() - try: - cls.process.wait(timeout=60) - except subprocess.TimeoutExpired: - pass terminate_and_kill_process_tree(cls.process, wait_timeout=60)