[CI] Graceful teardown for kv_canary and EAGLE spec fixtures (#32829)

python/sglang/test/chunked_prefill_test_utils.py

@@ -4,7 +4,6 @@ import time
 from types import SimpleNamespace
 from typing import ClassVar, List, Optional
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
@@ -14,6 +13,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
     try_cached_model,
 )
 
@@ -106,7 +106,7 @@ class ChunkedTestBase(ChunkedGsm8kMixin, CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if cls.process is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
 
 class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase):

python/sglang/test/kv_canary/e2e_base.py

@@ -8,7 +8,6 @@ from typing import ClassVar, Literal, Optional
 
 from sglang.srt.kv_canary.config import CanaryMode
 from sglang.srt.kv_canary.runner.swa_divergence import SwaDivergenceLog
-from sglang.srt.utils import kill_process_tree
 from sglang.test.kv_canary.mode_config import _MODE_CONFIGS, _ModeConfig
 from sglang.test.kv_canary.utils import build_canary_server_args, post_parallel_generate
 from sglang.test.kv_canary.violation_assert_mixin import CanaryViolationAssertMixin
@@ -17,6 +16,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # Long prompt body shared by all canary e2e tests. The repetition count is chosen
@@ -36,7 +36,7 @@ class CapturedServerE2EBase(CanaryViolationAssertMixin, CustomTestCase):
     @classmethod
     def tearDownClass(cls) -> None:
         if cls.process is not None:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         for buf in (cls._stdout_buf, cls._stderr_buf):
             if buf is not None:
                 buf.close()

python/sglang/test/server_fixtures/default_fixture.py

@@ -4,12 +4,12 @@ from contextlib import contextmanager
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ class DefaultServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
         time.sleep(2)
 
     @classmethod

python/sglang/test/server_fixtures/dsa_mtp_fixture.py

@@ -20,12 +20,12 @@ The base itself is NOT a runnable test (no `test_*` methods until a subclass
 mixes in the kits), so unittest discovery picks it up as empty.
 """
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 
@@ -95,4 +95,4 @@ class DsaMtpServerBase(CustomTestCase):
     @classmethod
     def tearDownClass(cls):
         if hasattr(cls, "process") and cls.process:
-            kill_process_tree(cls.process.pid)
+            terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/server_fixtures/eagle_fixture.py

@@ -4,7 +4,6 @@ import time
 
 import requests
 
-from sglang.srt.utils.common import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
     DEFAULT_TARGET_MODEL_EAGLE,
@@ -12,6 +11,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 PROMPTS = [
@@ -53,7 +53,7 @@ class EagleServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def send_request(self):
         time.sleep(random.uniform(0, 2))

python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py

@@ -12,7 +12,7 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import get_device_sm, kill_process_tree
+from sglang.srt.utils import get_device_sm
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -21,6 +21,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 GSM_DATASET_PATH = None
@@ -73,7 +74,7 @@ class TestHybridAttnBackendBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")

python/sglang/test/server_fixtures/mmmu_fixture.py

@@ -2,12 +2,12 @@ import logging
 import os
 import time
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ class MMMUServerBase(CustomTestCase):
     def tearDownClass(cls):
         if cls.process is not None and cls.process.poll() is None:
             try:
-                kill_process_tree(cls.process.pid, wait_timeout=60)
+                terminate_and_kill_process_tree(cls.process, wait_timeout=60)
             except Exception as e:
                 logger.error(f"Error killing process: {e}")
         time.sleep(2)

python/sglang/test/server_fixtures/ngram_fixture.py

@@ -15,13 +15,13 @@ mixes in GSM8KMixin), so unittest discovery picks it up as empty.
 """
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TARGET_MODEL_NGRAM,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 DEFAULT_NGRAM_SERVER_ARGS = [
@@ -71,4 +71,4 @@ class NgramServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/server_fixtures/pcg_spec_fixture.py

@@ -16,12 +16,12 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 
@@ -54,7 +54,7 @@ class PCGSpecBase:
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         eval_kwargs = dict(

python/sglang/test/server_fixtures/spec_eagle_fixture.py

@@ -17,7 +17,6 @@ import time
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils.common import kill_process_tree
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
@@ -28,6 +27,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # Chat-style prompts shared by send_request / send_requests_abort.
@@ -146,7 +146,7 @@ class SpecEagleServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid, wait_timeout=60)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     @property
     def tokenizer(self):

python/sglang/test/server_fixtures/standalone_fixture.py

@@ -15,7 +15,6 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_STANDALONE,
@@ -23,6 +22,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 GSM_DATASET_PATH = None
@@ -88,7 +88,7 @@ class StandaloneServerBase:
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")

python/sglang/test/server_fixtures/streaming_session_fixture.py

@@ -21,7 +21,6 @@ import aiohttp
 import requests
 
 from sglang.srt.environ import envs
-from sglang.srt.utils import kill_process_tree
 from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import (
     DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
@@ -29,6 +28,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 LOGPROB_PROMPTS = [
@@ -432,4 +432,4 @@ class StreamingSessionServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)

python/sglang/test/test_deterministic_utils.py

@@ -1,12 +1,12 @@
 import unittest
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_deterministic import BenchArgs, test_deterministic
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 DEFAULT_MODEL = "Qwen/Qwen3-8B"
@@ -43,7 +43,7 @@ class TestDeterministicBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def _extract_host_and_port(self, url):
         return url.split("://")[-1].split(":")[0], int(url.split(":")[-1])

python/sglang/test/test_utils.py

@@ -1064,6 +1064,27 @@ def popen_launch_server(
     raise TimeoutError(error_msg)
 
 
+def terminate_and_kill_process_tree(
+    process,
+    terminate_timeout: float = 60,
+    **kill_kwargs,
+) -> None:
+    """Shut a launched server down gracefully, then SIGKILL whatever is left.
+
+    A bare ``kill_process_tree`` leaves the kernel to unwind the CUDA context
+    and unpin the host memory during process reclaim, which can hold GPU memory
+    for minutes on a busy host -- long enough to trip the per-class GPU-idle
+    gate in the next ``setUpClass``. SIGTERM first so the server releases those
+    resources in userspace.
+    """
+    process.terminate()
+    try:
+        process.wait(timeout=terminate_timeout)
+    except subprocess.TimeoutExpired:
+        pass
+    kill_process_tree(process.pid, **kill_kwargs)
+
+
 def popen_launch_pd_server(
     model: str,
     base_url: str,

python/sglang/test/vlm_utils.py

@@ -8,12 +8,12 @@ import pybase64
 import requests
 from PIL import Image
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     popen_launch_server,
+    terminate_and_kill_process_tree,
 )
 
 # image
@@ -60,7 +60,7 @@ class TestOpenAIMLLMServerBase(CustomTestCase):
 
     @classmethod
     def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
+        terminate_and_kill_process_tree(cls.process, wait_timeout=60)
 
     def get_vision_request_kwargs(self):
         return self.get_request_kwargs()