[Bug Fix] Validate tokenizer-dependent features with skip_tokenizer_init (#27882)

Co-authored-by: Randall <randall@iterationlab.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

test/registered/unit/sampling/test_sampling_params.py

@@ -289,6 +289,17 @@ class TestSamplingParamsVerify(CustomTestCase):
 
 class TestSamplingParamsNormalize(CustomTestCase):
 
+    def _mock_tokenizer(self, encode_map=None):
+        """Create a mock tokenizer that returns predetermined token lists."""
+        tokenizer = MagicMock()
+        if encode_map:
+            tokenizer.encode.side_effect = (
+                lambda s, add_special_tokens=False: encode_map.get(s, [1])
+            )
+        else:
+            tokenizer.encode.return_value = [1]  # Default: 1 token
+        return tokenizer
+
     def test_none_stop_strs_becomes_empty_list(self):
         """Test that normalize() converts None stop to empty list with max_len=0."""
         sp = SamplingParams(stop=None)
@@ -299,20 +310,24 @@ class TestSamplingParamsNormalize(CustomTestCase):
     def test_string_stop_str_wrapped_in_list(self):
         """Test that normalize() wraps a single stop string into a list."""
         sp = SamplingParams(stop="<|end|>")
-        sp.normalize(tokenizer=None)
+        tokenizer = self._mock_tokenizer()
+        sp.normalize(tokenizer=tokenizer)
         self.assertEqual(sp.stop_strs, ["<|end|>"])
 
     def test_list_stop_strs_unchanged(self):
         """Test that normalize() preserves a list of stop strings as-is."""
         sp = SamplingParams(stop=["stop1", "stop2"])
-        sp.normalize(tokenizer=None)
+        tokenizer = self._mock_tokenizer()
+        sp.normalize(tokenizer=tokenizer)
         self.assertEqual(sp.stop_strs, ["stop1", "stop2"])
 
-    def test_stop_str_max_len_without_tokenizer(self):
-        """Test that without a tokenizer, max_len is the raw string character count."""
+    def test_stop_str_max_len_uses_encoded_length(self):
+        """Test that max_len is based on encoded token count, not character count."""
+        # "ab" encodes to 1 token, "cdef" encodes to 2 tokens
+        tokenizer = self._mock_tokenizer(encode_map={"ab": [1], "cdef": [2, 3]})
         sp = SamplingParams(stop=["ab", "cdef"])
-        sp.normalize(tokenizer=None)
-        self.assertEqual(sp.stop_str_max_len, 4)  # len("cdef")
+        sp.normalize(tokenizer=tokenizer)
+        self.assertEqual(sp.stop_str_max_len, 2)  # max token count
 
     def test_stop_str_max_len_with_tokenizer(self):
         """Test that with a tokenizer, max_len counts encoded token IDs."""
@@ -336,13 +351,15 @@ class TestSamplingParamsNormalize(CustomTestCase):
     def test_string_stop_regex_wrapped_in_list(self):
         """Test that normalize() wraps a single stop_regex string into a list."""
         sp = SamplingParams(stop_regex=r"\d+")
-        sp.normalize(tokenizer=None)
+        tokenizer = self._mock_tokenizer()
+        sp.normalize(tokenizer=tokenizer)
         self.assertEqual(sp.stop_regex_strs, [r"\d+"])
 
     def test_stop_regex_max_len_computed(self):
         """Test that bounded regex computes a finite max length."""
         sp = SamplingParams(stop_regex=r"[a-z]{3}")
-        sp.normalize(tokenizer=None)
+        tokenizer = self._mock_tokenizer()
+        sp.normalize(tokenizer=tokenizer)
         self.assertEqual(sp.stop_regex_max_len, 3)