diff --git a/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py b/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py new file mode 100644 index 0000000000..eddfc4fcff --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py @@ -0,0 +1,293 @@ +r"""Encode a long rendered chat prompt in parallel with ids identical to +``tokenizer.encode(text, **encode_kwargs)``. + +A HF fast tokenizer first splits the text at added tokens and runs the +normalizer, pre-tokenizer and model on each piece in between independently. +Cutting the text right after an added-token match therefore cannot change the +ids, so the pieces go through the backend's ``encode_batch``, which runs on the +tokenizers (rayon) thread pool. + +A single message is often most of the prompt, so long gaps between added +tokens are also cut at pre-token boundaries, but only for the exact +Split-regex + ByteLevel + BPE pipeline of GLM / Llama-3 style tokenizers +(``_INTRA_SPLIT_REGEX``). For that regex, a match containing a letter ends at +the last letter of a run (only the ``'s``-style and ``?\p{L}+`` alternatives +contain letters) and a match containing ``\n`` never continues into a letter, +so "letter|space letter" and "\n|letter" are always pre-token boundaries. +Truncating a chunk after a letter cannot change any match; after ``\n`` the +only lookahead that could flip (``\s+(?!\S)``) is shadowed by the earlier +``\s*[\r\n]+`` alternative. BPE never merges across pre-tokens. + +Any configuration where that argument does not hold falls back to the original +encode: slow or wrapped tokenizers, specials added by the post-processor, +post-processors that can rewrite sequence ids, lstrip / rstrip / single_word +added tokens, pre-tokenizers that treat the first piece of a sequence +differently (Metaspace), split_special_tokens, and truncation or padding left +enabled on the backend. + +Gated by SGLANG_PARALLEL_PROMPT_ENCODE and SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS. +""" + +import json +import logging +import os +import re +import time +import weakref +from itertools import chain + +from sglang.srt.environ import envs + +logger = logging.getLogger(__name__) + +# Adjacent pieces are merged up to this size; tiny inputs only add per-input +# overhead in encode_batch. +_MIN_CHUNK_CHARS = 2048 +_MAX_CHUNKS = 64 + +# Pre-tokenizers that act on each piece independently of its position in the +# sequence. Metaspace (prepend_scheme="first") and anything unknown are not. +_SAFE_PRE_TOKENIZERS = { + "ByteLevel", + "Split", + "Digits", + "Punctuation", + "Whitespace", + "WhitespaceSplit", + "BertPreTokenizer", + "CharDelimiterSplit", + "UnicodeScripts", +} + +_ID_PRESERVING_POST_PROCESSORS = { + "BertProcessing", + "ByteLevel", + "RobertaProcessing", +} + +# Pre-token regex for which the intra-gap cut points below were verified. +_INTRA_SPLIT_REGEX = ( + r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}|" + r" ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" +) +# ASCII letters only: Python and Oniguruma agree on them for \p{L}. +_INTRA_CUT = re.compile(r"(?<=[A-Za-z])(?= [A-Za-z])|(?<=\n)(?=[A-Za-z])") + +_plans = weakref.WeakKeyDictionary() +# None: not yet checked; False: encode_batch consumed only one CPU (rayon +# disabled, e.g. after a fork). The PID prevents inheriting the parent's result. +_parallel_pid = os.getpid() +_parallel_ok = None + + +def _pre_tokenizer_safe(cfg): + if cfg is None: + return True + if cfg.get("type") == "Sequence": + return all(_pre_tokenizer_safe(c) for c in cfg.get("pretokenizers", [])) + return cfg.get("type") in _SAFE_PRE_TOKENIZERS + + +def _template_post_processor_safe(cfg): + sequences = 0 + for item in cfg.get("single", []): + if not isinstance(item, dict) or len(item) != 1: + return False + if isinstance(item.get("SpecialToken"), dict): + continue + sequence = item.get("Sequence") + if not isinstance(sequence, dict) or sequence.get("id") != "A": + return False + sequences += 1 + return sequences == 1 + + +def _post_processor_safe(cfg): + if cfg is None: + return True + processor_type = cfg.get("type") + if processor_type == "Sequence": + return all(_post_processor_safe(c) for c in cfg.get("processors", [])) + if processor_type == "TemplateProcessing": + return _template_post_processor_safe(cfg) + return processor_type in _ID_PRESERVING_POST_PROCESSORS + + +def _intra_cut_safe(cfg): + pre = cfg.get("pre_tokenizer") or {} + subs = pre.get("pretokenizers") if pre.get("type") == "Sequence" else None + if not subs or len(subs) != 2: + return False + split, bl = subs + model = cfg.get("model") or {} + return ( + cfg.get("normalizer") is None + and split.get("type") == "Split" + and split.get("pattern") == {"Regex": _INTRA_SPLIT_REGEX} + and split.get("behavior") == "Isolated" + and not split.get("invert") + and bl.get("type") == "ByteLevel" + and not bl.get("add_prefix_space") + and not bl.get("use_regex") + and model.get("type") == "BPE" + and model.get("dropout") is None + ) + + +def _alternation(tokens): + if not tokens: + return None + # Longest first: Python's leftmost-first alternation then equals the + # backend's leftmost-longest added-token match. + alts = sorted(set(tokens), key=len, reverse=True) + return re.compile("|".join(re.escape(t) for t in alts)) + + +def _build_plan(tokenizer): + """Return (backend, specials_added, pattern, normalized_pattern, intra) + or None.""" + try: + from transformers import PreTrainedTokenizerBase + except ImportError: + return None + backend = getattr(tokenizer, "_tokenizer", None) + if not getattr(tokenizer, "is_fast", False) or backend is None: + return None + if not hasattr(backend, "encode_batch") or not hasattr(backend, "to_str"): + return None + # Only the stock HF fast path: encode -> _encode_plus -> backend.encode_batch. + if type(tokenizer).encode is not PreTrainedTokenizerBase.encode: + return None + if "encode" in vars(tokenizer) or "_encode_plus" in vars(tokenizer): + return None + enc_plus = type(tokenizer)._encode_plus + if enc_plus.__module__ != "transformers.tokenization_utils_tokenizers": + return None + if getattr(tokenizer, "split_special_tokens", False): + return None + try: + cfg = json.loads(backend.to_str()) + except Exception: + return None + if not _pre_tokenizer_safe(cfg.get("pre_tokenizer")): + return None + if not _post_processor_safe(cfg.get("post_processor")): + return None + added = backend.get_added_tokens_decoder().values() + if any(t.lstrip or t.rstrip or t.single_word for t in added): + return None + raw = [t.content for t in added if not t.normalized and t.content] + normed = [t.content for t in added if t.normalized and t.content] + pattern = _alternation(raw) + # Normalized added tokens are matched on normalized text in a second pass. + # Without a normalizer that text is the original, so their ends are cut + # points too; with one, only the first-pass (raw) tokens are used. + npattern = _alternation(normed) if backend.normalizer is None else None + intra = _INTRA_CUT if _intra_cut_safe(cfg) else None + if pattern is None and npattern is None and intra is None: + return None + pp = backend.post_processor + specials_added = pp.num_special_tokens_to_add(False) if pp is not None else 0 + return backend, specials_added, pattern, npattern, intra + + +def _plan_for(tokenizer): + try: + return _plans[tokenizer] + except KeyError: + pass + except TypeError: + return _build_plan(tokenizer) + plan = _build_plan(tokenizer) + _plans[tokenizer] = plan + return plan + + +def _added_spans(text, pattern, npattern): + """(start, end) of the backend's added-token matches (first pass, then + second pass inside the gaps), in ascending order.""" + if pattern is None: + if npattern is None: + return [] + return [m.span() for m in npattern.finditer(text)] + spans, pos = [], 0 + for m in pattern.finditer(text): + if npattern is not None: + spans.extend(m2.span() for m2 in npattern.finditer(text, pos, m.start())) + spans.append(m.span()) + pos = m.end() + if npattern is not None: + spans.extend(m2.span() for m2 in npattern.finditer(text, pos)) + return spans + + +def split_prompt(text, pattern, npattern, intra=None, max_chunks=_MAX_CHUNKS): + """Split text into at most ~max_chunks pieces at added-token ends and, if + intra is given, at its matches strictly inside the gaps between them.""" + n = len(text) + target = max(_MIN_CHUNK_CHARS, -(-n // max_chunks)) + chunks, start, pos = [], 0, 0 + for s, e in _added_spans(text, pattern, npattern) + [(n, n)]: + # Gap [pos, s): both characters around an intra cut must lie in it. + while intra is not None: + m = intra.search(text, max(start + target, pos + 1), s) + if m is None: + break + chunks.append(text[start : m.start()]) + start = m.start() + if e - start >= target and e < n: + chunks.append(text[start:e]) + start = e + pos = e + chunks.append(text[start:]) + return chunks + + +def parallel_prompt_encode(tokenizer, text, encode_kwargs=None): + """Drop-in for ``tokenizer.encode(text, **encode_kwargs)``.""" + global _parallel_ok, _parallel_pid + pid = os.getpid() + if pid != _parallel_pid: + _parallel_pid = pid + _parallel_ok = None + encode_kwargs = encode_kwargs or {} + if ( + _parallel_ok is False + or not envs.SGLANG_PARALLEL_PROMPT_ENCODE.get() + or not isinstance(text, str) + or len(text) < envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.get() + or set(encode_kwargs) - {"add_special_tokens"} + ): + return tokenizer.encode(text, **encode_kwargs) + plan = _plan_for(tokenizer) + if plan is None: + return tokenizer.encode(text, **encode_kwargs) + backend, specials_added, pattern, npattern, intra = plan + if encode_kwargs.get("add_special_tokens", True) and specials_added: + return tokenizer.encode(text, **encode_kwargs) + if ( + backend.truncation is not None + or backend.padding is not None + or backend.encode_special_tokens + ): + return tokenizer.encode(text, **encode_kwargs) + chunks = split_prompt(text, pattern, npattern, intra) + if len(chunks) < 2: + return tokenizer.encode(text, **encode_kwargs) + + check = _parallel_ok is None + if check: + w0, c0 = time.perf_counter(), time.process_time() + encodings = backend.encode_batch(chunks, add_special_tokens=False) + if check: + wall, cpu = time.perf_counter() - w0, time.process_time() - c0 + # Rayon worker CPU adds up well beyond wall time. Serial stays near 1.0; + # 1.5 leaves headroom for scheduler and measurement noise. + _parallel_ok = not (wall > 0.005 and cpu < 1.5 * wall) + if not _parallel_ok: + logger.warning( + "parallel_prompt_encode: tokenizers encode_batch runs serially in " + "this process (TOKENIZERS_PARALLELISM off or process forked after " + "tokenizers used its pool); using the original encode." + ) + return list(chain.from_iterable(e.ids for e in encodings)) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 04f30d2f8f..6c19793192 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -46,6 +46,9 @@ from sglang.srt.entrypoints.openai import ( encoding_dsv32, encoding_dsv41, ) +from sglang.srt.entrypoints.openai.parallel_prompt_encode import ( + parallel_prompt_encode, +) from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionMessageContentTextPart, ChatCompletionMessageContentVideoPart, @@ -1836,8 +1839,8 @@ class OpenAIServingChat(OpenAIServingBase): return_dict=False, **template_kwargs, ) - prompt_ids = self.tokenizer_manager.tokenizer.encode( - rendered_prompt, **encode_kwargs + prompt_ids = parallel_prompt_encode( + self.tokenizer_manager.tokenizer, rendered_prompt, encode_kwargs ) decoded_prompt = ( self.tokenizer_manager.tokenizer.decode(prompt_ids) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 7283e0a280..5befef1ff8 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1514,6 +1514,11 @@ class Envs: # Think tokens budget: negative means unlimited, >= 0 caps thinking tokens SGLANG_MAX_THINK_TOKENS = EnvInt(-1) SGLANG_PATCH_TOKENIZER = EnvBool(True) + # Encode long rendered chat prompts as chunks on the tokenizers thread pool. + SGLANG_PARALLEL_PROMPT_ENCODE = EnvBool(True) + # Shorter prompts use the single-call encode; below this the gain is eaten + # by rayon dispatch and the id merge. + SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS = EnvInt(32768) SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4) SGLANG_DEFAULT_THINKING = EnvBool(False) diff --git a/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py b/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py new file mode 100644 index 0000000000..980e7ac262 --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py @@ -0,0 +1,593 @@ +"""Chunked prompt encoding must produce the ids the single encode produces. + +`parallel_prompt_encode` cuts a rendered prompt and sends the pieces through +`backend.encode_batch`, so its whole contract is that the concatenated ids are +identical to `tokenizer.encode(text, **encode_kwargs)` — an off-by-one cut +point is not a crash, it is a silently different prompt. These cases pin the +identity on a ByteLevel-BPE tokenizer built in memory with the exact +Split-regex + ByteLevel + BPE pipeline the intra-gap cuts were verified for, +and pin the refusals: every tokenizer configuration outside that pipeline, and +every gate, must reach the original encode instead. + +The tests never assert that the rayon pool actually ran the chunks in +parallel. The helper probes for a serial `encode_batch` once and then disables +itself for the process, which is a correctness-preserving decision a CPU +runner is free to make; `_parallel_ok` is pinned per test so the id +comparisons do not depend on it. +""" + +import random +import sys +import unittest +import unittest.mock +import weakref +from functools import lru_cache + +import pytest +from tokenizers import ( + AddedToken, + Regex, + Tokenizer, + decoders, + models, + normalizers, + pre_tokenizers, + processors, + trainers, +) +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast + +from sglang.srt.entrypoints.openai import parallel_prompt_encode as ppe +from sglang.srt.environ import envs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +_TRAIN_CORPUS = [ + "the quick brown fox jumps over the lazy dog", + "hello world, this is a test of the tokenizer", + "def main(argv):\n return 0\n", + "it's they're I'm we'll you'd she's", + "中文文本 日本語 한국어 émigré naïve Ωmega", + "numbers 123 4567 89 1,234.56 [bracket]", +] +_ALL_KWARGS = ({}, {"add_special_tokens": True}, {"add_special_tokens": False}) +_SPECIALS = [ + "<|user|>", + "<|assistant|>", + "", + "", + "", + "[gMASK]", + "", + "/nothink", +] + + +@lru_cache(maxsize=1) +def _safe_backend_json(): + """A GLM / Llama-3 shaped backend: the pipeline `_intra_cut_safe` accepts. + + Only the pre-tokenizer pipeline and the model type decide which cut points + the helper takes, so a 900-token vocabulary exercises the same code path a + real checkpoint does. The ByteLevel alphabet is seeded explicitly so every + byte of the test corpus has a token and nothing is silently dropped. + """ + backend = Tokenizer(models.BPE()) + backend.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split( + Regex(ppe._INTRA_SPLIT_REGEX), behavior="isolated", invert=False + ), + pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False), + ] + ) + backend.decoder = decoders.ByteLevel() + backend.train_from_iterator( + _TRAIN_CORPUS * 20, + trainers.BpeTrainer( + vocab_size=900, + show_progress=False, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), + special_tokens=[], + ), + ) + backend.add_tokens( + [AddedToken(t, normalized=False, special=True) for t in _SPECIALS] + ) + return backend.to_str() + + +def _tokenizer(mutate=None, **kwargs): + backend = Tokenizer.from_str(_safe_backend_json()) + if mutate is not None: + mutate(backend) + return PreTrainedTokenizerFast(tokenizer_object=backend, **kwargs) + + +def _chunk_ids(tokenizer, text, all_cuts=False): + """Encode `text` chunk by chunk, the way the helper splits it. + + Each chunk goes through the backend on this thread, so the comparison is + the plan's cut points against a single encode — not `encode_batch`'s + threading. + """ + backend, _, pattern, npattern, intra = ppe._plan_for(tokenizer) + if all_cuts: + # Target size 1: take every candidate cut point, not one per 2 KiB. + with unittest.mock.patch.object(ppe, "_MIN_CHUNK_CHARS", 1): + chunks = ppe.split_prompt( + text, pattern, npattern, intra, max_chunks=1 << 30 + ) + else: + chunks = ppe.split_prompt(text, pattern, npattern, intra) + ids = [] + for chunk in chunks: + ids.extend(backend.encode(chunk, add_special_tokens=False).ids) + return ids, chunks + + +_EDGE_TEXTS = [ + "", + "a", + " ", + "\n", + "abc def", + "no specials at all\nline two\r\nline3 foo", + "<|user|>", + "<|user|><|assistant|>", + "<|user|>x<|assistant|>", + "<|user|> hello world", + "hello world <|assistant|>", + "[gMASK]<|user|>\nhi<|assistant|>\nreasoninganswer", + "/nothinking", + "/nothinker", + "", + "<>", + "中文文本,没有空格。日本語のテキスト 한국어", + "emoji 🙂🙃 x 👩‍👩‍👧 é é \u200b x", + "tabs\tand double spaces\n\n\nX\n \nY \n Z", + "it's they're I'M we'll 'd x's", + "\r\nA\rB\n\rC", + "123456789 abc123def 1,234.56", + "word Line Para", + "Ωmega ΣΙΓΜΑ ßtraße İi", + "a" * 5000 + " b", + "x y\n" * 3000, +] + + +def _fuzz_texts(): + """Random texts over the atoms that sit on either side of a cut point.""" + rng = random.Random(0) + atoms = list("aZ é中🙂'1 \n\r\t.,;:{}()<>/_-\"\\") + [ + "\u0301", + " ", + "'s", + " ", + "\r\n", + " \n", + "\n ", + ] + atoms += _SPECIALS + texts = [ + "".join(rng.choice(atoms) for _ in range(rng.randint(0, 400))) + for _ in range(4000) + ] + texts += [ + "".join(rng.choice(atoms) for _ in range(rng.randint(5000, 60000))) + for _ in range(30) + ] + return texts + + +class _EncodeCase(CustomTestCase): + def setUp(self): + super().setUp() + # `_plans` and `_parallel_ok` are process-wide and sticky: a plan + # cached for a mutated tokenizer, or a probe that fell back, would + # leak into whatever case runs next. + self.addCleanup(setattr, ppe, "_plans", ppe._plans) + ppe._plans = weakref.WeakKeyDictionary() + self.addCleanup(setattr, ppe, "_parallel_ok", ppe._parallel_ok) + ppe._parallel_ok = True + + def _set_min_chars(self, value): + override = envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.override(value) + override.__enter__() + self.addCleanup(override.__exit__, None, None, None) + + def _no_min_chars(self): + self._set_min_chars(0) + + def _assert_matches_encode(self, tokenizer, text, kwargs_variants=_ALL_KWARGS): + for encode_kwargs in kwargs_variants: + self.assertEqual( + ppe.parallel_prompt_encode(tokenizer, text, encode_kwargs), + tokenizer.encode(text, **encode_kwargs), + f"{encode_kwargs} on {text[:60]!r}", + ) + + +class TestSafePipelineIdsAreIdentical(_EncodeCase): + def setUp(self): + super().setUp() + self._no_min_chars() + self.tokenizer = _tokenizer() + + def test_the_plan_takes_every_kind_of_cut_point(self): + """Without this the id comparisons below could pass on no cuts at all.""" + plan = ppe._plan_for(self.tokenizer) + self.assertIsNotNone(plan) + self.assertIsNotNone(plan[2], "added tokens must be matched") + self.assertIs(plan[4], ppe._INTRA_CUT, "intra-gap cuts must be enabled") + for text, expected in ( + ("abcd", ["", "abcd"]), + ("abcd efgh", ["abcd", " efgh"]), + ("abcd\nefgh", ["abcd\n", "efgh"]), + ): + self.assertEqual( + _chunk_ids(self.tokenizer, text, all_cuts=True)[1], expected + ) + + def test_edge_cases_encode_the_same_at_every_cut_point(self): + texts = list(_EDGE_TEXTS) + for text in _EDGE_TEXTS: + texts += [ + "" + text, + text + "", + "" + text + "", + ] + for text in texts: + reference = self.tokenizer.encode(text) + self.assertEqual( + _chunk_ids(self.tokenizer, text, all_cuts=True)[0], reference + ) + self._assert_matches_encode(self.tokenizer, text) + + def test_fuzzed_texts_encode_the_same_at_every_cut_point(self): + for text in _fuzz_texts(): + reference = self.tokenizer.encode(text) + self.assertEqual( + _chunk_ids(self.tokenizer, text, all_cuts=True)[0], + reference, + repr(text[:80]), + ) + self.assertEqual( + ppe.parallel_prompt_encode(self.tokenizer, text, {}), reference + ) + + def test_a_long_prompt_uses_the_default_chunking(self): + text = ( + "<|user|>\n" + "the quick brown fox jumps over the lazy dog\n" * 40 + ) * 60 + ids, chunks = _chunk_ids(self.tokenizer, text) + self.assertGreater(len(chunks), 1) + self.assertLessEqual(len(chunks), ppe._MAX_CHUNKS + 1) + self.assertEqual(ids, self.tokenizer.encode(text)) + self._assert_matches_encode(self.tokenizer, text) + + def test_the_serial_probe_leaves_the_ids_unchanged(self): + """The first call times `encode_batch`; whichever way that goes, the + ids are the ones `encode` returns.""" + ppe._parallel_ok = None + text = "hello world, this is a test\n" * 4000 + self.assertEqual( + ppe.parallel_prompt_encode(self.tokenizer, text, {}), + self.tokenizer.encode(text), + ) + self.assertIn(ppe._parallel_ok, (True, False)) + + def _run_probe(self, process_cpu, thread_cpu): + ppe._parallel_ok = None + text = "hello world, this is a test\n" * 4000 + with ( + unittest.mock.patch.object( + ppe.time, "perf_counter", side_effect=(1.0, 1.01) + ), + unittest.mock.patch.object( + ppe.time, "thread_time", side_effect=(1.0, 1.0 + thread_cpu) + ), + unittest.mock.patch.object( + ppe.time, + "process_time", + side_effect=(1.0, 1.0 + process_cpu), + create=True, + ), + ): + ppe.parallel_prompt_encode(self.tokenizer, text, {}) + return ppe._parallel_ok + + def test_the_probe_counts_rayon_worker_cpu(self): + self.assertTrue(self._run_probe(process_cpu=0.04, thread_cpu=0.006)) + + def test_the_probe_rejects_one_cpu(self): + self.assertFalse(self._run_probe(process_cpu=0.01, thread_cpu=0.0)) + + def test_a_forked_process_rechecks_parallelism(self): + old_pid = getattr(ppe, "_parallel_pid", None) + self.addCleanup(setattr, ppe, "_parallel_pid", old_pid) + ppe._parallel_ok = False + ppe._parallel_pid = 100 + text = "hello world, this is a test\n" * 4000 + with ( + unittest.mock.patch.object(ppe.os, "getpid", return_value=101), + unittest.mock.patch.object( + ppe.time, "perf_counter", side_effect=(1.0, 1.01) + ), + unittest.mock.patch.object( + ppe.time, "process_time", side_effect=(1.0, 1.04) + ), + ): + ppe.parallel_prompt_encode(self.tokenizer, text, {}) + self.assertEqual(ppe._parallel_pid, 101) + self.assertTrue(ppe._parallel_ok) + + +def _metaspace(backend): + backend.pre_tokenizer = pre_tokenizers.Metaspace() + + +def _other_split_regex(backend): + backend.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split(Regex(r"\s+|\w+|[^\w\s]+"), behavior="isolated"), + pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False), + ] + ) + + +def _prefix_space(backend): + backend.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split(Regex(ppe._INTRA_SPLIT_REGEX), behavior="isolated"), + pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False), + ] + ) + + +def _lowercase_normalizer(backend): + backend.add_tokens([AddedToken("NORM", normalized=True)]) + backend.normalizer = normalizers.Lowercase() + + +def _post_processor_adds_bos(backend): + backend.post_processor = processors.TemplateProcessing( + single=" $A", + special_tokens=[("", backend.token_to_id(""))], + ) + + +def _post_processor_duplicates_sequence(backend): + backend.post_processor = processors.TemplateProcessing( + single="$A $A", special_tokens=[] + ) + + +def _post_processor_sequence(backend, duplicate=False): + template = "$A $A" if duplicate else " $A" + backend.post_processor = processors.Sequence( + [ + processors.ByteLevel(), + processors.TemplateProcessing( + single=template, + special_tokens=[("", backend.token_to_id(""))], + ), + ] + ) + + +class _WrappedTokenizer(PreTrainedTokenizerFast): + def encode(self, text, **kwargs): + return super().encode(text, **kwargs) + + +class _SlowWordTokenizer(PreTrainedTokenizer): + """A whitespace tokenizer on the python backend, i.e. `is_fast` False.""" + + def __init__(self, **kwargs): + self._vocab = { + w: i for i, w in enumerate(["", "hello", "world", "a", "b"]) + } + super().__init__(unk_token="", **kwargs) + + @property + def vocab_size(self): + return len(self._vocab) + + def get_vocab(self): + return dict(self._vocab) + + def _tokenize(self, text, **kwargs): + return text.split() + + def _convert_token_to_id(self, token): + return self._vocab.get(token, self._vocab[""]) + + def _convert_id_to_token(self, index): + return list(self._vocab)[index] + + +class TestUnsafeTokenizersFallBack(_EncodeCase): + def setUp(self): + super().setUp() + self._no_min_chars() + self.text = ( + "Hello NORM world\nsecond line, it's fine tail " * 200 + ) + + def _assert_original_encode(self, tokenizer, kwargs_variants=_ALL_KWARGS): + with unittest.mock.patch.object( + ppe, "split_prompt", side_effect=AssertionError("chunked path taken") + ): + self._assert_matches_encode(tokenizer, self.text, kwargs_variants) + + def test_configurations_without_a_plan(self): + cases = { + "Metaspace pre-tokenizer": _tokenizer(_metaspace), + "lstrip added token": _tokenizer( + lambda b: b.add_tokens( + [AddedToken("", lstrip=True, normalized=False)] + ) + ), + "rstrip added token": _tokenizer( + lambda b: b.add_tokens( + [AddedToken("", rstrip=True, normalized=False)] + ) + ), + "single_word added token": _tokenizer( + lambda b: b.add_tokens( + [AddedToken("", single_word=True, normalized=False)] + ) + ), + "split_special_tokens": _tokenizer(split_special_tokens=True), + "wrapped encode": _WrappedTokenizer( + tokenizer_object=Tokenizer.from_str(_safe_backend_json()) + ), + "slow tokenizer": _SlowWordTokenizer(), + } + for name, tokenizer in cases.items(): + with self.subTest(name): + self.assertIsNone(ppe._plan_for(tokenizer)) + self._assert_original_encode(tokenizer) + + def test_configurations_that_keep_only_the_added_token_cuts(self): + """A safe-but-unverified pre-tokenizer still cuts at added tokens: the + backend restarts the pipeline there whatever the pre-tokenizer is.""" + cases = { + "different Split regex": _tokenizer(_other_split_regex), + "ByteLevel add_prefix_space": _tokenizer(_prefix_space), + "normalizer present": _tokenizer(_lowercase_normalizer), + } + for name, tokenizer in cases.items(): + with self.subTest(name): + plan = ppe._plan_for(tokenizer) + self.assertIsNotNone(plan) + self.assertIsNone(plan[4], "intra-gap cuts are not verified here") + self.assertEqual( + _chunk_ids(tokenizer, self.text, all_cuts=True)[0], + tokenizer.encode(self.text, add_special_tokens=False), + ) + self._assert_matches_encode(tokenizer, self.text) + + def test_a_post_processor_that_adds_specials_falls_back_when_asked_for_them(self): + tokenizer = _tokenizer(_post_processor_adds_bos) + plan = ppe._plan_for(tokenizer) + self.assertIsNotNone(plan) + self.assertEqual(plan[1], 1) + with unittest.mock.patch.object( + ppe, "split_prompt", side_effect=AssertionError("chunked path taken") + ): + for encode_kwargs in ({}, {"add_special_tokens": True}): + self.assertEqual( + ppe.parallel_prompt_encode(tokenizer, self.text, encode_kwargs), + tokenizer.encode(self.text, **encode_kwargs), + ) + # Asked not to add them, the chunked path is exact again. + encode_kwargs = {"add_special_tokens": False} + self.assertEqual( + ppe.parallel_prompt_encode(tokenizer, self.text, encode_kwargs), + tokenizer.encode(self.text, **encode_kwargs), + ) + self.assertEqual( + _chunk_ids(tokenizer, self.text, all_cuts=True)[0], + tokenizer.encode(self.text, add_special_tokens=False), + ) + + def test_a_post_processor_that_duplicates_ids_falls_back(self): + tok = _tokenizer(_post_processor_duplicates_sequence) + text = "hello world " + "hello world " * 2000 + self.assertIsNone(ppe._plan_for(tok)) + with unittest.mock.patch.object( + ppe, "split_prompt", side_effect=AssertionError("chunked path taken") + ): + assert ppe.parallel_prompt_encode( + tok, text, {"add_special_tokens": False} + ) == tok.encode(text, add_special_tokens=False) + + def test_sequence_post_processors_are_checked_recursively(self): + safe = _tokenizer(lambda b: _post_processor_sequence(b)) + unsafe = _tokenizer(lambda b: _post_processor_sequence(b, duplicate=True)) + self.assertIsNotNone(ppe._plan_for(safe)) + self.assertIsNone(ppe._plan_for(unsafe)) + self._assert_original_encode(unsafe) + + def test_backend_state_that_rewrites_the_ids_falls_back(self): + """These three live on the backend, not in the plan, so the helper has + to look at them per call. + + The mutation is applied after the wrapper is built (it copies the + backend), and each case gets one call: falling back runs + `tokenizer.encode`, which clears the backend's truncation and padding. + """ + cases = { + "truncation": lambda b: b.enable_truncation(50), + "padding": lambda b: b.enable_padding(length=64), + "encode_special_tokens": lambda b: setattr( + b, "encode_special_tokens", True + ), + } + for name, mutate in cases.items(): + with self.subTest(name): + tokenizer = _tokenizer() + mutate(tokenizer._tokenizer) + self.assertIsNotNone(ppe._plan_for(tokenizer)) + self._assert_original_encode(tokenizer, ({},)) + + +class TestGatesAndArgumentAllowlist(_EncodeCase): + threshold = 4096 + + def setUp(self): + super().setUp() + # Pinned rather than inherited: the ambient environment may carry its + # own threshold, and the default is 32 KiB of text per case. + self._set_min_chars(self.threshold) + self.tokenizer = _tokenizer() + self.text = "the quick brown fox jumps over the lazy dog\n" * 200 + self.assertGreater(len(self.text), self.threshold) + + def _assert_chunked(self, encode_kwargs): + with unittest.mock.patch.object( + ppe, "split_prompt", wraps=ppe.split_prompt + ) as split: + ids = ppe.parallel_prompt_encode(self.tokenizer, self.text, encode_kwargs) + self.assertEqual(split.call_count, 1) + self.assertEqual(ids, self.tokenizer.encode(self.text, **(encode_kwargs or {}))) + + def _assert_original(self, text, encode_kwargs): + with unittest.mock.patch.object( + ppe, "split_prompt", side_effect=AssertionError("chunked path taken") + ): + ids = ppe.parallel_prompt_encode(self.tokenizer, text, encode_kwargs) + self.assertEqual(ids, self.tokenizer.encode(text, **(encode_kwargs or {}))) + + def test_a_long_prompt_takes_the_chunked_path(self): + self._assert_chunked(None) + self._assert_chunked({"add_special_tokens": True}) + self._assert_chunked({"add_special_tokens": False}) + + def test_a_prompt_below_the_char_threshold_uses_the_original_encode(self): + self._assert_original(self.text[: self.threshold - 1], None) + with envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.override(len(self.text) + 1): + self._assert_original(self.text, None) + + def test_the_env_switch_disables_the_chunked_path(self): + with envs.SGLANG_PARALLEL_PROMPT_ENCODE.override(False): + self._assert_original(self.text, None) + + def test_encode_kwargs_outside_the_allowlist_use_the_original_encode(self): + self._assert_original(self.text, {"truncation": False}) + self._assert_original( + self.text, {"add_special_tokens": False, "truncation": False} + ) + + def test_a_process_with_serial_encode_batch_stays_on_the_original_encode(self): + ppe._parallel_ok = False + self._assert_original(self.text, None) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))