[CI] Cut repeated tokenizer loads, serial subprocesses and a double scan (#36241)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alex Nails
2026-08-24 18:22:43 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 68575b23d0
commit 5ffdb02d0c
3 changed files with 118 additions and 120 deletions
@@ -1,3 +1,4 @@
import functools
import json
import unittest
import warnings
@@ -33,10 +34,17 @@ from sglang.srt.function_call.pythonic_detector import PythonicDetector
from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
register_cpu_ci(est_time=70, suite="base-c-test-cpu")
@functools.lru_cache(maxsize=None)
def _shared_tokenizer(path: str):
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
return get_tokenizer(path)
class TestInklingDetector(unittest.TestCase):
def setUp(self):
self.tools = [
@@ -1599,9 +1607,7 @@ class TestDeepSeekV32Detector(unittest.TestCase):
),
]
self.detector = DeepSeekV32Detector()
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
self.tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3.2")
self.tokenizer = _shared_tokenizer("deepseek-ai/DeepSeek-V3.2")
self.interval = 1
def test_detect_and_parse_xml_format(self):
@@ -2049,9 +2055,7 @@ class TestDeepSeekV4Detector(unittest.TestCase):
),
]
self.detector = DeepSeekV4Detector()
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
self.tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3.2")
self.tokenizer = _shared_tokenizer("deepseek-ai/DeepSeek-V3.2")
self.interval = 1
def test_detect_and_parse_xml_format(self):