[CI] Graceful teardown for kv_canary and EAGLE spec fixtures (#32829)

python/sglang/test/chunked_prefill_test_utils.py

@@ -4,7 +4,6 @@ import time
 from types import SimpleNamespace
 from typing import ClassVar, List, Optional
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
@@ -14,6 +13,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
     try_cached_model,
 )
 
@@ -106,7 +106,7 @@ class ChunkedTestBase(ChunkedGsm8kMixin, CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if cls.process is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
 
 class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase):

python/sglang/test/kv_canary/e2e_base.py

@@ -8,7 +8,6 @@ from typing import ClassVar, Literal, Optional
 
 from sglang.srt.kv_canary.config import CanaryMode
 from sglang.srt.kv_canary.runner.swa_divergence import SwaDivergenceLog
-from sglang.srt.utils import kill_process_tree
 from sglang.test.kv_canary.mode_config import _MODE_CONFIGS, _ModeConfig
 from sglang.test.kv_canary.utils import build_canary_server_args, post_parallel_generate
 from sglang.test.kv_canary.violation_assert_mixin import CanaryViolationAssertMixin
@@ -17,6 +16,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # Long prompt body shared by all canary e2e tests. The repetition count is chosen
@@ -36,7 +36,7 @@ class CapturedServerE2EBase(CanaryViolationAssertMixin, CustomTestCase):
     @classmethod
     def tearDownClass(cls) -> None:
         if cls.process is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         for buf in (cls._stdout_buf, cls._stderr_buf):
             if buf is not None:
                 buf.close()

python/sglang/test/server_fixtures/default_fixture.py

@@ -4,12 +4,12 @@ from contextlib import contextmanager
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ class DefaultServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         time.sleep(2)
 
     @classmethod

python/sglang/test/server_fixtures/dsa_mtp_fixture.py

@@ -20,12 +20,12 @@ The base itself is NOT a runnable test (no `test_*` methods until a subclass
 mixes in the kits), so unittest discovery picks it up as empty.
 """
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 
@@ -95,4 +95,4 @@ class DsaMtpServerBase(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if hasattr(cls, "process") and cls.process:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/server_fixtures/eagle_fixture.py

@@ -4,7 +4,6 @@ import time
 
 import requests
 
-from sglang.srt.utils.common import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
     DEFAULT_TARGET_MODEL_EAGLE,
@@ -12,6 +11,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 PROMPTS = [
@@ -53,7 +53,7 @@ class EagleServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def send_request(self):
         time.sleep(random.uniform(0, 2))

python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py

@@ -12,7 +12,7 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import get_device_sm, kill_process_tree
+from sglang.srt.utils import get_device_sm
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -21,6 +21,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 GSM_DATASET_PATH = None
@@ -73,7 +74,7 @@ class TestHybridAttnBackendBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")

python/sglang/test/server_fixtures/mmmu_fixture.py

@@ -2,12 +2,12 @@ import logging
 import os
 import time
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ class MMMUServerBase(CustomTestCase):
     def tearDownClass(cls):
         if cls.process is not None and cls.process.poll() is None:
             try:
-                kill_process_tree(cls.process.pid, wait_timeout=60)
+                terminate_and_kill_process_tree(cls.process, wait_timeout=60)
             except Exception as e:
                 logger.error(f"Error killing process: {e}")
         time.sleep(2)

python/sglang/test/server_fixtures/ngram_fixture.py

@@ -15,13 +15,13 @@ mixes in GSM8KMixin), so unittest discovery picks it up as empty.
 """
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TARGET_MODEL_NGRAM,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 DEFAULT_NGRAM_SERVER_ARGS = [
@@ -71,4 +71,4 @@ class NgramServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/server_fixtures/pcg_spec_fixture.py

@@ -16,12 +16,12 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 
@@ -54,7 +54,7 @@ class PCGSpecBase:
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         eval_kwargs = dict(

python/sglang/test/server_fixtures/spec_eagle_fixture.py

@@ -17,7 +17,6 @@ import time
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils.common import kill_process_tree
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
@@ -28,6 +27,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # Chat-style prompts shared by send_request / send_requests_abort.
@@ -146,7 +146,7 @@ class SpecEagleServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     @property
     def tokenizer(self):

python/sglang/test/server_fixtures/standalone_fixture.py

@@ -15,7 +15,6 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_STANDALONE,
@@ -23,6 +22,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 GSM_DATASET_PATH = None
@@ -88,7 +88,7 @@ class StandaloneServerBase:
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")

python/sglang/test/server_fixtures/streaming_session_fixture.py

@@ -21,7 +21,6 @@ import aiohttp
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
@@ -29,6 +28,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 LOGPROB_PROMPTS = [
@@ -432,4 +432,4 @@ class StreamingSessionServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/test_deterministic_utils.py

@@ -1,12 +1,12 @@
 import unittest
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_deterministic import BenchArgs, test_deterministic
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 DEFAULT_MODEL = "Qwen/Qwen3-8B"
@@ -43,7 +43,7 @@ class TestDeterministicBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def _extract_host_and_port(self, url):
         return url.split("://")[-1].split(":")[0], int(url.split(":")[-1])

python/sglang/test/test_utils.py

@@ -1064,6 +1064,27 @@ def popen_launch_server(
     raise TimeoutError(error_msg)
 
 
+def terminate_and_kill_process_tree(
+    process,
+    terminate_timeout: float = 60,
+    **kill_kwargs,
+) -> None:
+    """Shut a launched server down gracefully, then SIGKILL whatever is left.
+
+    A bare ``kill_process_tree`` leaves the kernel to unwind the CUDA context
+    and unpin the host memory during process reclaim, which can hold GPU memory
+    for minutes on a busy host -- long enough to trip the per-class GPU-idle
+    gate in the next ``setUpClass``. SIGTERM first so the server releases those
+    resources in userspace.
+    """
+    process.terminate()
+    try:
+        process.wait(timeout=terminate_timeout)
+    except subprocess.TimeoutExpired:
+        pass
+    kill_process_tree(process.pid, **kill_kwargs)
+
+
 def popen_launch_pd_server(
     model: str,
     base_url: str,

python/sglang/test/vlm_utils.py

@@ -8,12 +8,12 @@ import pybase64
 import requests
 from PIL import Image
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # image
@@ -60,7 +60,7 @@ class TestOpenAIMLLMServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def get_vision_request_kwargs(self):
         return self.get_request_kwargs()

test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py

@@ -1,12 +1,10 @@
 import os
 import shutil
-import subprocess
 import tempfile
 import unittest
 
 from test_unified_radix_cache_kl_nightly import AccuracyTwoPassMixin
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin
 from sglang.test.kl_multiturn_utils import (
@@ -19,6 +17,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-h100")
@@ -69,7 +68,7 @@ class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
 
 # ─── Mamba + HiCache L2 ──────────────────────────────────────────────────────
@@ -128,12 +127,7 @@ class TestUnifiedMambaHiCache(UnifiedRadixTreeTestMixin, CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls.process.terminate()
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
-        try:
-            cls.process.wait(timeout=60)
-        except subprocess.TimeoutExpired:
-            pass
-        kill_process_tree(cls.process.pid)
 
 
 # ─── Mamba + HiCache L3 (file backend) ───────────────────────────────────────
@@ -194,12 +188,7 @@ class TestUnifiedMambaHiCacheL3(AccuracyTwoPassMixin, CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls.process.terminate()
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
-        try:
-            cls.process.wait(timeout=60)
-        except subprocess.TimeoutExpired:
-            pass
-        kill_process_tree(cls.process.pid)
         if os.path.isdir(cls.hicache_dir):
             shutil.rmtree(cls.hicache_dir, ignore_errors=True)
 

test/registered/vlm/test_vision_openai_server_a.py

@@ -17,6 +17,7 @@ from sglang.test.vlm_utils import (
     OmniOpenAITestMixin,
     TestOpenAIMLLMServerBase,
     VideoOpenAITestMixin,
+    terminate_and_kill_process_tree,
 )
 
 register_cuda_ci(est_time=780, stage="base-b", runner_config="1-gpu-large")
@@ -74,7 +75,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_single_image_chat_completion(self):
         client = openai.Client(api_key=self.api_key, base_url=self.base_url)

test/registered/xpu/test_gemma_4_e2b.py

@@ -29,8 +29,8 @@ from sglang.srt.utils.common import is_xpu
 from sglang.test.test_utils import CustomTestCase
 from sglang.test.vlm_utils import (
     DEFAULT_URL_FOR_TEST,
-    kill_process_tree,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 MODEL = "google/gemma-4-E2B-it"
@@ -98,7 +98,7 @@ class TestGemma4E2BXPU(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         _empty_xpu_cache()
 
     def test_simple_qa(self):