[CI] Cut repeated tokenizer loads, serial subprocesses and a double scan (#36241)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
68575b23d0
commit
5ffdb02d0c
@@ -1,3 +1,4 @@
|
||||
import functools
|
||||
import json
|
||||
import unittest
|
||||
import warnings
|
||||
@@ -33,10 +34,17 @@ from sglang.srt.function_call.pythonic_detector import PythonicDetector
|
||||
from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=70, suite="base-c-test-cpu")
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _shared_tokenizer(path: str):
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
|
||||
return get_tokenizer(path)
|
||||
|
||||
|
||||
class TestInklingDetector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tools = [
|
||||
@@ -1599,9 +1607,7 @@ class TestDeepSeekV32Detector(unittest.TestCase):
|
||||
),
|
||||
]
|
||||
self.detector = DeepSeekV32Detector()
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
|
||||
self.tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3.2")
|
||||
self.tokenizer = _shared_tokenizer("deepseek-ai/DeepSeek-V3.2")
|
||||
self.interval = 1
|
||||
|
||||
def test_detect_and_parse_xml_format(self):
|
||||
@@ -2049,9 +2055,7 @@ class TestDeepSeekV4Detector(unittest.TestCase):
|
||||
),
|
||||
]
|
||||
self.detector = DeepSeekV4Detector()
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
|
||||
self.tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3.2")
|
||||
self.tokenizer = _shared_tokenizer("deepseek-ai/DeepSeek-V3.2")
|
||||
self.interval = 1
|
||||
|
||||
def test_detect_and_parse_xml_format(self):
|
||||
|
||||
Reference in New Issue
Block a user