diff --git a/python/sglang/srt/sampling/sampling_params.py b/python/sglang/srt/sampling/sampling_params.py index 38a950596..b5a2ce262 100644 --- a/python/sglang/srt/sampling/sampling_params.py +++ b/python/sglang/srt/sampling/sampling_params.py @@ -29,6 +29,36 @@ TOP_K_ALL = 1 << 30 logger = logging.getLogger(__name__) +def raise_if_tokenizer_required( + tokenizer, stop_strs, stop_regex_strs, min_new_tokens=0 +): + """Raise ValueError if tokenizer-dependent features are used without a tokenizer. + + String-based stop conditions (stop_strs, stop_regex_strs) require tokenizer.decode() + to convert output token IDs to text for matching. min_new_tokens requires the + tokenizer's eos_token_id to penalize. When skip_tokenizer_init=True, these cannot + be used. + """ + if tokenizer is not None: + return + + if stop_strs: + raise ValueError( + f"stop={stop_strs!r} is unavailable when skip_tokenizer_init=True " + "(requires tokenizer to decode tokens to text for matching)." + ) + if stop_regex_strs: + raise ValueError( + f"stop_regex={stop_regex_strs!r} is unavailable when skip_tokenizer_init=True " + "(requires tokenizer to decode tokens to text for matching)." + ) + if min_new_tokens > 0: + raise ValueError( + f"min_new_tokens={min_new_tokens} is unavailable when skip_tokenizer_init=True " + "(requires tokenizer for eos_token_id)." + ) + + class SamplingParams: """ The sampling parameters. @@ -210,6 +240,11 @@ class SamplingParams: self.stop_regex_max_len = stop_regex_max_len + # Validate tokenizer is available for tokenizer-dependent features + raise_if_tokenizer_required( + tokenizer, self.stop_strs, self.stop_regex_strs, self.min_new_tokens + ) + # This function gets a strict upperbound on the maximum number of tokens that would need # to be buffered to match the input regex string diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 20d9e427d..399449c5d 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -4354,6 +4354,11 @@ class ServerArgs: ) self.enable_dynamic_batch_tokenizer = False + logger.info( + "skip_tokenizer_init=True: string-based stop conditions (stop, stop_regex) " + "and min_new_tokens are unavailable." + ) + def _handle_environment_variables(self): envs.SGLANG_ENABLE_TORCH_COMPILE.set("1" if self.enable_torch_compile else "0") if self.mamba_ssm_dtype is not None: @@ -4681,6 +4686,14 @@ class ServerArgs: self.preferred_sampling_params ) + # Validate preferred_sampling_params doesn't use tokenizer-dependent features + if self.skip_tokenizer_init: + from sglang.srt.sampling.sampling_params import SamplingParams + + test_params = SamplingParams(**self.preferred_sampling_params) + # raises if tokenizer-dependent features used + test_params.normalize(None) + def _handle_crash_dump_env(self): if not self.crash_dump_folder: return diff --git a/test/registered/unit/managers/test_stop_str_speculative.py b/test/registered/unit/managers/test_stop_str_speculative.py index 351d84cd1..c1bd117d3 100644 --- a/test/registered/unit/managers/test_stop_str_speculative.py +++ b/test/registered/unit/managers/test_stop_str_speculative.py @@ -41,9 +41,16 @@ class _FakeTokenizer: return "".join(ID_TO_TEXT[int(i)] for i in ids) +class _MockTokenizerForNormalize: + """Mock tokenizer for normalize() - returns char-count as token list.""" + + def encode(self, s, add_special_tokens=False): + return list(range(len(s))) # One "token" per character + + def _make_req(output_ids, stop=None, stop_regex=None): sp = SamplingParams(max_new_tokens=1000, stop=stop, stop_regex=stop_regex) - sp.normalize(tokenizer=None) # char-based stop_str_max_len + sp.normalize(tokenizer=_MockTokenizerForNormalize()) # char-based stop_str_max_len req = Req( rid="t", origin_input_text="", diff --git a/test/registered/unit/sampling/test_sampling_params.py b/test/registered/unit/sampling/test_sampling_params.py index caf94c449..2b1236cfd 100644 --- a/test/registered/unit/sampling/test_sampling_params.py +++ b/test/registered/unit/sampling/test_sampling_params.py @@ -289,6 +289,17 @@ class TestSamplingParamsVerify(CustomTestCase): class TestSamplingParamsNormalize(CustomTestCase): + def _mock_tokenizer(self, encode_map=None): + """Create a mock tokenizer that returns predetermined token lists.""" + tokenizer = MagicMock() + if encode_map: + tokenizer.encode.side_effect = ( + lambda s, add_special_tokens=False: encode_map.get(s, [1]) + ) + else: + tokenizer.encode.return_value = [1] # Default: 1 token + return tokenizer + def test_none_stop_strs_becomes_empty_list(self): """Test that normalize() converts None stop to empty list with max_len=0.""" sp = SamplingParams(stop=None) @@ -299,20 +310,24 @@ class TestSamplingParamsNormalize(CustomTestCase): def test_string_stop_str_wrapped_in_list(self): """Test that normalize() wraps a single stop string into a list.""" sp = SamplingParams(stop="<|end|>") - sp.normalize(tokenizer=None) + tokenizer = self._mock_tokenizer() + sp.normalize(tokenizer=tokenizer) self.assertEqual(sp.stop_strs, ["<|end|>"]) def test_list_stop_strs_unchanged(self): """Test that normalize() preserves a list of stop strings as-is.""" sp = SamplingParams(stop=["stop1", "stop2"]) - sp.normalize(tokenizer=None) + tokenizer = self._mock_tokenizer() + sp.normalize(tokenizer=tokenizer) self.assertEqual(sp.stop_strs, ["stop1", "stop2"]) - def test_stop_str_max_len_without_tokenizer(self): - """Test that without a tokenizer, max_len is the raw string character count.""" + def test_stop_str_max_len_uses_encoded_length(self): + """Test that max_len is based on encoded token count, not character count.""" + # "ab" encodes to 1 token, "cdef" encodes to 2 tokens + tokenizer = self._mock_tokenizer(encode_map={"ab": [1], "cdef": [2, 3]}) sp = SamplingParams(stop=["ab", "cdef"]) - sp.normalize(tokenizer=None) - self.assertEqual(sp.stop_str_max_len, 4) # len("cdef") + sp.normalize(tokenizer=tokenizer) + self.assertEqual(sp.stop_str_max_len, 2) # max token count def test_stop_str_max_len_with_tokenizer(self): """Test that with a tokenizer, max_len counts encoded token IDs.""" @@ -336,13 +351,15 @@ class TestSamplingParamsNormalize(CustomTestCase): def test_string_stop_regex_wrapped_in_list(self): """Test that normalize() wraps a single stop_regex string into a list.""" sp = SamplingParams(stop_regex=r"\d+") - sp.normalize(tokenizer=None) + tokenizer = self._mock_tokenizer() + sp.normalize(tokenizer=tokenizer) self.assertEqual(sp.stop_regex_strs, [r"\d+"]) def test_stop_regex_max_len_computed(self): """Test that bounded regex computes a finite max length.""" sp = SamplingParams(stop_regex=r"[a-z]{3}") - sp.normalize(tokenizer=None) + tokenizer = self._mock_tokenizer() + sp.normalize(tokenizer=tokenizer) self.assertEqual(sp.stop_regex_max_len, 3)