diff --git a/python/sglang/test/chunked_prefill_test_utils.py b/python/sglang/test/chunked_prefill_test_utils.py index ddcaf1333..d1a86b0d7 100644 --- a/python/sglang/test/chunked_prefill_test_utils.py +++ b/python/sglang/test/chunked_prefill_test_utils.py @@ -4,7 +4,6 @@ import time from types import SimpleNamespace from typing import ClassVar, List, Optional -from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, @@ -14,6 +13,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, try_cached_model, ) @@ -106,7 +106,7 @@ class ChunkedTestBase(ChunkedGsm8kMixin, CustomTestCase): @classmethod def tearDownClass(cls): if cls.process is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase): diff --git a/python/sglang/test/kv_canary/e2e_base.py b/python/sglang/test/kv_canary/e2e_base.py index 6d76bf047..788576766 100644 --- a/python/sglang/test/kv_canary/e2e_base.py +++ b/python/sglang/test/kv_canary/e2e_base.py @@ -8,7 +8,6 @@ from typing import ClassVar, Literal, Optional from sglang.srt.kv_canary.config import CanaryMode from sglang.srt.kv_canary.runner.swa_divergence import SwaDivergenceLog -from sglang.srt.utils import kill_process_tree from sglang.test.kv_canary.mode_config import _MODE_CONFIGS, _ModeConfig from sglang.test.kv_canary.utils import build_canary_server_args, post_parallel_generate from sglang.test.kv_canary.violation_assert_mixin import CanaryViolationAssertMixin @@ -17,6 +16,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) # Long prompt body shared by all canary e2e tests. The repetition count is chosen @@ -36,7 +36,7 @@ class CapturedServerE2EBase(CanaryViolationAssertMixin, CustomTestCase): @classmethod def tearDownClass(cls) -> None: if cls.process is not None: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) for buf in (cls._stdout_buf, cls._stderr_buf): if buf is not None: buf.close() diff --git a/python/sglang/test/server_fixtures/default_fixture.py b/python/sglang/test/server_fixtures/default_fixture.py index d10b72658..0d170bb5b 100644 --- a/python/sglang/test/server_fixtures/default_fixture.py +++ b/python/sglang/test/server_fixtures/default_fixture.py @@ -4,12 +4,12 @@ from contextlib import contextmanager import requests -from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ class DefaultServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid, wait_timeout=60) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) time.sleep(2) @classmethod diff --git a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py index c04821d39..127c83ec0 100644 --- a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py +++ b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py @@ -20,12 +20,12 @@ The base itself is NOT a runnable test (no `test_*` methods until a subclass mixes in the kits), so unittest discovery picks it up as empty. """ -from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) @@ -95,4 +95,4 @@ class DsaMtpServerBase(CustomTestCase): @classmethod def tearDownClass(cls): if hasattr(cls, "process") and cls.process: - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) diff --git a/python/sglang/test/server_fixtures/eagle_fixture.py b/python/sglang/test/server_fixtures/eagle_fixture.py index 991585202..510c94d21 100644 --- a/python/sglang/test/server_fixtures/eagle_fixture.py +++ b/python/sglang/test/server_fixtures/eagle_fixture.py @@ -4,7 +4,6 @@ import time import requests -from sglang.srt.utils.common import kill_process_tree from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, DEFAULT_TARGET_MODEL_EAGLE, @@ -12,6 +11,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) PROMPTS = [ @@ -53,7 +53,7 @@ class EagleServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid, wait_timeout=60) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def send_request(self): time.sleep(random.uniform(0, 2)) diff --git a/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py index 774020530..9aba6ee48 100644 --- a/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py +++ b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py @@ -12,7 +12,7 @@ from types import SimpleNamespace import requests from sglang.srt.environ import envs -from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.srt.utils import get_device_sm from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -21,6 +21,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) GSM_DATASET_PATH = None @@ -73,7 +74,7 @@ class TestHybridAttnBackendBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_gsm8k(self): requests.get(self.base_url + "/flush_cache") diff --git a/python/sglang/test/server_fixtures/mmmu_fixture.py b/python/sglang/test/server_fixtures/mmmu_fixture.py index 8070eff66..f203a5c6f 100644 --- a/python/sglang/test/server_fixtures/mmmu_fixture.py +++ b/python/sglang/test/server_fixtures/mmmu_fixture.py @@ -2,12 +2,12 @@ import logging import os import time -from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ class MMMUServerBase(CustomTestCase): def tearDownClass(cls): if cls.process is not None and cls.process.poll() is None: try: - kill_process_tree(cls.process.pid, wait_timeout=60) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) except Exception as e: logger.error(f"Error killing process: {e}") time.sleep(2) diff --git a/python/sglang/test/server_fixtures/ngram_fixture.py b/python/sglang/test/server_fixtures/ngram_fixture.py index 2dfc18fdb..f05aaaf92 100644 --- a/python/sglang/test/server_fixtures/ngram_fixture.py +++ b/python/sglang/test/server_fixtures/ngram_fixture.py @@ -15,13 +15,13 @@ mixes in GSM8KMixin), so unittest discovery picks it up as empty. """ from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TARGET_MODEL_NGRAM, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) DEFAULT_NGRAM_SERVER_ARGS = [ @@ -71,4 +71,4 @@ class NgramServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) diff --git a/python/sglang/test/server_fixtures/pcg_spec_fixture.py b/python/sglang/test/server_fixtures/pcg_spec_fixture.py index c0307dfe3..3bcfd2f61 100644 --- a/python/sglang/test/server_fixtures/pcg_spec_fixture.py +++ b/python/sglang/test/server_fixtures/pcg_spec_fixture.py @@ -16,12 +16,12 @@ from types import SimpleNamespace import requests -from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, popen_launch_server, + terminate_and_kill_process_tree, ) @@ -54,7 +54,7 @@ class PCGSpecBase: @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_gsm8k(self): eval_kwargs = dict( diff --git a/python/sglang/test/server_fixtures/spec_eagle_fixture.py b/python/sglang/test/server_fixtures/spec_eagle_fixture.py index 3af28f703..4ca00b001 100644 --- a/python/sglang/test/server_fixtures/spec_eagle_fixture.py +++ b/python/sglang/test/server_fixtures/spec_eagle_fixture.py @@ -17,7 +17,6 @@ import time import requests from sglang.srt.environ import envs -from sglang.srt.utils.common import kill_process_tree from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, @@ -28,6 +27,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) # Chat-style prompts shared by send_request / send_requests_abort. @@ -146,7 +146,7 @@ class SpecEagleServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid, wait_timeout=60) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) @property def tokenizer(self): diff --git a/python/sglang/test/server_fixtures/standalone_fixture.py b/python/sglang/test/server_fixtures/standalone_fixture.py index 12dcbfd13..f02fe4c02 100644 --- a/python/sglang/test/server_fixtures/standalone_fixture.py +++ b/python/sglang/test/server_fixtures/standalone_fixture.py @@ -15,7 +15,6 @@ from types import SimpleNamespace import requests from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_STANDALONE, @@ -23,6 +22,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, popen_launch_server, + terminate_and_kill_process_tree, ) GSM_DATASET_PATH = None @@ -88,7 +88,7 @@ class StandaloneServerBase: @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_gsm8k(self): requests.get(self.base_url + "/flush_cache") diff --git a/python/sglang/test/server_fixtures/streaming_session_fixture.py b/python/sglang/test/server_fixtures/streaming_session_fixture.py index 6c95548ed..fe7b8ca9b 100644 --- a/python/sglang/test/server_fixtures/streaming_session_fixture.py +++ b/python/sglang/test/server_fixtures/streaming_session_fixture.py @@ -21,7 +21,6 @@ import aiohttp import requests from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.test_utils import ( DEFAULT_SMALL_MODEL_NAME_FOR_TEST, @@ -29,6 +28,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) LOGPROB_PROMPTS = [ @@ -432,4 +432,4 @@ class StreamingSessionServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) diff --git a/python/sglang/test/test_deterministic_utils.py b/python/sglang/test/test_deterministic_utils.py index c1bd308f9..c1e89e8eb 100644 --- a/python/sglang/test/test_deterministic_utils.py +++ b/python/sglang/test/test_deterministic_utils.py @@ -1,12 +1,12 @@ import unittest -from sglang.srt.utils import kill_process_tree from sglang.test.test_deterministic import BenchArgs, test_deterministic from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) DEFAULT_MODEL = "Qwen/Qwen3-8B" @@ -43,7 +43,7 @@ class TestDeterministicBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def _extract_host_and_port(self, url): return url.split("://")[-1].split(":")[0], int(url.split(":")[-1]) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 7a6b1f641..2f12b7d1e 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -1064,6 +1064,27 @@ def popen_launch_server( raise TimeoutError(error_msg) +def terminate_and_kill_process_tree( + process, + terminate_timeout: float = 60, + **kill_kwargs, +) -> None: + """Shut a launched server down gracefully, then SIGKILL whatever is left. + + A bare ``kill_process_tree`` leaves the kernel to unwind the CUDA context + and unpin the host memory during process reclaim, which can hold GPU memory + for minutes on a busy host -- long enough to trip the per-class GPU-idle + gate in the next ``setUpClass``. SIGTERM first so the server releases those + resources in userspace. + """ + process.terminate() + try: + process.wait(timeout=terminate_timeout) + except subprocess.TimeoutExpired: + pass + kill_process_tree(process.pid, **kill_kwargs) + + def popen_launch_pd_server( model: str, base_url: str, diff --git a/python/sglang/test/vlm_utils.py b/python/sglang/test/vlm_utils.py index a1897be0c..b476e3381 100644 --- a/python/sglang/test/vlm_utils.py +++ b/python/sglang/test/vlm_utils.py @@ -8,12 +8,12 @@ import pybase64 import requests from PIL import Image -from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) # image @@ -60,7 +60,7 @@ class TestOpenAIMLLMServerBase(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def get_vision_request_kwargs(self): return self.get_request_kwargs() diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py index 5465cb32f..3ee63f27f 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py @@ -1,12 +1,10 @@ import os import shutil -import subprocess import tempfile import unittest from test_unified_radix_cache_kl_nightly import AccuracyTwoPassMixin -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin from sglang.test.kl_multiturn_utils import ( @@ -19,6 +17,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-h100") @@ -69,7 +68,7 @@ class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) # ─── Mamba + HiCache L2 ────────────────────────────────────────────────────── @@ -128,12 +127,7 @@ class TestUnifiedMambaHiCache(UnifiedRadixTreeTestMixin, CustomTestCase): @classmethod def tearDownClass(cls): - cls.process.terminate() - try: - cls.process.wait(timeout=60) - except subprocess.TimeoutExpired: - pass - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) # ─── Mamba + HiCache L3 (file backend) ─────────────────────────────────────── @@ -194,12 +188,7 @@ class TestUnifiedMambaHiCacheL3(AccuracyTwoPassMixin, CustomTestCase): @classmethod def tearDownClass(cls): - cls.process.terminate() - try: - cls.process.wait(timeout=60) - except subprocess.TimeoutExpired: - pass - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) if os.path.isdir(cls.hicache_dir): shutil.rmtree(cls.hicache_dir, ignore_errors=True) diff --git a/test/registered/vlm/test_vision_openai_server_a.py b/test/registered/vlm/test_vision_openai_server_a.py index a553368f3..00d546f23 100644 --- a/test/registered/vlm/test_vision_openai_server_a.py +++ b/test/registered/vlm/test_vision_openai_server_a.py @@ -17,6 +17,7 @@ from sglang.test.vlm_utils import ( OmniOpenAITestMixin, TestOpenAIMLLMServerBase, VideoOpenAITestMixin, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=780, stage="base-b", runner_config="1-gpu-large") @@ -74,7 +75,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) def test_single_image_chat_completion(self): client = openai.Client(api_key=self.api_key, base_url=self.base_url) diff --git a/test/registered/xpu/test_gemma_4_e2b.py b/test/registered/xpu/test_gemma_4_e2b.py index ebe1e2dd2..a340a58bc 100644 --- a/test/registered/xpu/test_gemma_4_e2b.py +++ b/test/registered/xpu/test_gemma_4_e2b.py @@ -29,8 +29,8 @@ from sglang.srt.utils.common import is_xpu from sglang.test.test_utils import CustomTestCase from sglang.test.vlm_utils import ( DEFAULT_URL_FOR_TEST, - kill_process_tree, popen_launch_server, + terminate_and_kill_process_tree, ) MODEL = "google/gemma-4-E2B-it" @@ -98,7 +98,7 @@ class TestGemma4E2BXPU(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + terminate_and_kill_process_tree(cls.process, wait_timeout=60) _empty_xpu_cache() def test_simple_qa(self):