diff --git a/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py b/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py
new file mode 100644
index 0000000000..eddfc4fcff
--- /dev/null
+++ b/python/sglang/srt/entrypoints/openai/parallel_prompt_encode.py
@@ -0,0 +1,293 @@
+r"""Encode a long rendered chat prompt in parallel with ids identical to
+``tokenizer.encode(text, **encode_kwargs)``.
+
+A HF fast tokenizer first splits the text at added tokens and runs the
+normalizer, pre-tokenizer and model on each piece in between independently.
+Cutting the text right after an added-token match therefore cannot change the
+ids, so the pieces go through the backend's ``encode_batch``, which runs on the
+tokenizers (rayon) thread pool.
+
+A single message is often most of the prompt, so long gaps between added
+tokens are also cut at pre-token boundaries, but only for the exact
+Split-regex + ByteLevel + BPE pipeline of GLM / Llama-3 style tokenizers
+(``_INTRA_SPLIT_REGEX``). For that regex, a match containing a letter ends at
+the last letter of a run (only the ``'s``-style and ``?\p{L}+`` alternatives
+contain letters) and a match containing ``\n`` never continues into a letter,
+so "letter|space letter" and "\n|letter" are always pre-token boundaries.
+Truncating a chunk after a letter cannot change any match; after ``\n`` the
+only lookahead that could flip (``\s+(?!\S)``) is shadowed by the earlier
+``\s*[\r\n]+`` alternative. BPE never merges across pre-tokens.
+
+Any configuration where that argument does not hold falls back to the original
+encode: slow or wrapped tokenizers, specials added by the post-processor,
+post-processors that can rewrite sequence ids, lstrip / rstrip / single_word
+added tokens, pre-tokenizers that treat the first piece of a sequence
+differently (Metaspace), split_special_tokens, and truncation or padding left
+enabled on the backend.
+
+Gated by SGLANG_PARALLEL_PROMPT_ENCODE and SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.
+"""
+
+import json
+import logging
+import os
+import re
+import time
+import weakref
+from itertools import chain
+
+from sglang.srt.environ import envs
+
+logger = logging.getLogger(__name__)
+
+# Adjacent pieces are merged up to this size; tiny inputs only add per-input
+# overhead in encode_batch.
+_MIN_CHUNK_CHARS = 2048
+_MAX_CHUNKS = 64
+
+# Pre-tokenizers that act on each piece independently of its position in the
+# sequence. Metaspace (prepend_scheme="first") and anything unknown are not.
+_SAFE_PRE_TOKENIZERS = {
+ "ByteLevel",
+ "Split",
+ "Digits",
+ "Punctuation",
+ "Whitespace",
+ "WhitespaceSplit",
+ "BertPreTokenizer",
+ "CharDelimiterSplit",
+ "UnicodeScripts",
+}
+
+_ID_PRESERVING_POST_PROCESSORS = {
+ "BertProcessing",
+ "ByteLevel",
+ "RobertaProcessing",
+}
+
+# Pre-token regex for which the intra-gap cut points below were verified.
+_INTRA_SPLIT_REGEX = (
+ r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}|"
+ r" ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+)
+# ASCII letters only: Python and Oniguruma agree on them for \p{L}.
+_INTRA_CUT = re.compile(r"(?<=[A-Za-z])(?= [A-Za-z])|(?<=\n)(?=[A-Za-z])")
+
+_plans = weakref.WeakKeyDictionary()
+# None: not yet checked; False: encode_batch consumed only one CPU (rayon
+# disabled, e.g. after a fork). The PID prevents inheriting the parent's result.
+_parallel_pid = os.getpid()
+_parallel_ok = None
+
+
+def _pre_tokenizer_safe(cfg):
+ if cfg is None:
+ return True
+ if cfg.get("type") == "Sequence":
+ return all(_pre_tokenizer_safe(c) for c in cfg.get("pretokenizers", []))
+ return cfg.get("type") in _SAFE_PRE_TOKENIZERS
+
+
+def _template_post_processor_safe(cfg):
+ sequences = 0
+ for item in cfg.get("single", []):
+ if not isinstance(item, dict) or len(item) != 1:
+ return False
+ if isinstance(item.get("SpecialToken"), dict):
+ continue
+ sequence = item.get("Sequence")
+ if not isinstance(sequence, dict) or sequence.get("id") != "A":
+ return False
+ sequences += 1
+ return sequences == 1
+
+
+def _post_processor_safe(cfg):
+ if cfg is None:
+ return True
+ processor_type = cfg.get("type")
+ if processor_type == "Sequence":
+ return all(_post_processor_safe(c) for c in cfg.get("processors", []))
+ if processor_type == "TemplateProcessing":
+ return _template_post_processor_safe(cfg)
+ return processor_type in _ID_PRESERVING_POST_PROCESSORS
+
+
+def _intra_cut_safe(cfg):
+ pre = cfg.get("pre_tokenizer") or {}
+ subs = pre.get("pretokenizers") if pre.get("type") == "Sequence" else None
+ if not subs or len(subs) != 2:
+ return False
+ split, bl = subs
+ model = cfg.get("model") or {}
+ return (
+ cfg.get("normalizer") is None
+ and split.get("type") == "Split"
+ and split.get("pattern") == {"Regex": _INTRA_SPLIT_REGEX}
+ and split.get("behavior") == "Isolated"
+ and not split.get("invert")
+ and bl.get("type") == "ByteLevel"
+ and not bl.get("add_prefix_space")
+ and not bl.get("use_regex")
+ and model.get("type") == "BPE"
+ and model.get("dropout") is None
+ )
+
+
+def _alternation(tokens):
+ if not tokens:
+ return None
+ # Longest first: Python's leftmost-first alternation then equals the
+ # backend's leftmost-longest added-token match.
+ alts = sorted(set(tokens), key=len, reverse=True)
+ return re.compile("|".join(re.escape(t) for t in alts))
+
+
+def _build_plan(tokenizer):
+ """Return (backend, specials_added, pattern, normalized_pattern, intra)
+ or None."""
+ try:
+ from transformers import PreTrainedTokenizerBase
+ except ImportError:
+ return None
+ backend = getattr(tokenizer, "_tokenizer", None)
+ if not getattr(tokenizer, "is_fast", False) or backend is None:
+ return None
+ if not hasattr(backend, "encode_batch") or not hasattr(backend, "to_str"):
+ return None
+ # Only the stock HF fast path: encode -> _encode_plus -> backend.encode_batch.
+ if type(tokenizer).encode is not PreTrainedTokenizerBase.encode:
+ return None
+ if "encode" in vars(tokenizer) or "_encode_plus" in vars(tokenizer):
+ return None
+ enc_plus = type(tokenizer)._encode_plus
+ if enc_plus.__module__ != "transformers.tokenization_utils_tokenizers":
+ return None
+ if getattr(tokenizer, "split_special_tokens", False):
+ return None
+ try:
+ cfg = json.loads(backend.to_str())
+ except Exception:
+ return None
+ if not _pre_tokenizer_safe(cfg.get("pre_tokenizer")):
+ return None
+ if not _post_processor_safe(cfg.get("post_processor")):
+ return None
+ added = backend.get_added_tokens_decoder().values()
+ if any(t.lstrip or t.rstrip or t.single_word for t in added):
+ return None
+ raw = [t.content for t in added if not t.normalized and t.content]
+ normed = [t.content for t in added if t.normalized and t.content]
+ pattern = _alternation(raw)
+ # Normalized added tokens are matched on normalized text in a second pass.
+ # Without a normalizer that text is the original, so their ends are cut
+ # points too; with one, only the first-pass (raw) tokens are used.
+ npattern = _alternation(normed) if backend.normalizer is None else None
+ intra = _INTRA_CUT if _intra_cut_safe(cfg) else None
+ if pattern is None and npattern is None and intra is None:
+ return None
+ pp = backend.post_processor
+ specials_added = pp.num_special_tokens_to_add(False) if pp is not None else 0
+ return backend, specials_added, pattern, npattern, intra
+
+
+def _plan_for(tokenizer):
+ try:
+ return _plans[tokenizer]
+ except KeyError:
+ pass
+ except TypeError:
+ return _build_plan(tokenizer)
+ plan = _build_plan(tokenizer)
+ _plans[tokenizer] = plan
+ return plan
+
+
+def _added_spans(text, pattern, npattern):
+ """(start, end) of the backend's added-token matches (first pass, then
+ second pass inside the gaps), in ascending order."""
+ if pattern is None:
+ if npattern is None:
+ return []
+ return [m.span() for m in npattern.finditer(text)]
+ spans, pos = [], 0
+ for m in pattern.finditer(text):
+ if npattern is not None:
+ spans.extend(m2.span() for m2 in npattern.finditer(text, pos, m.start()))
+ spans.append(m.span())
+ pos = m.end()
+ if npattern is not None:
+ spans.extend(m2.span() for m2 in npattern.finditer(text, pos))
+ return spans
+
+
+def split_prompt(text, pattern, npattern, intra=None, max_chunks=_MAX_CHUNKS):
+ """Split text into at most ~max_chunks pieces at added-token ends and, if
+ intra is given, at its matches strictly inside the gaps between them."""
+ n = len(text)
+ target = max(_MIN_CHUNK_CHARS, -(-n // max_chunks))
+ chunks, start, pos = [], 0, 0
+ for s, e in _added_spans(text, pattern, npattern) + [(n, n)]:
+ # Gap [pos, s): both characters around an intra cut must lie in it.
+ while intra is not None:
+ m = intra.search(text, max(start + target, pos + 1), s)
+ if m is None:
+ break
+ chunks.append(text[start : m.start()])
+ start = m.start()
+ if e - start >= target and e < n:
+ chunks.append(text[start:e])
+ start = e
+ pos = e
+ chunks.append(text[start:])
+ return chunks
+
+
+def parallel_prompt_encode(tokenizer, text, encode_kwargs=None):
+ """Drop-in for ``tokenizer.encode(text, **encode_kwargs)``."""
+ global _parallel_ok, _parallel_pid
+ pid = os.getpid()
+ if pid != _parallel_pid:
+ _parallel_pid = pid
+ _parallel_ok = None
+ encode_kwargs = encode_kwargs or {}
+ if (
+ _parallel_ok is False
+ or not envs.SGLANG_PARALLEL_PROMPT_ENCODE.get()
+ or not isinstance(text, str)
+ or len(text) < envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.get()
+ or set(encode_kwargs) - {"add_special_tokens"}
+ ):
+ return tokenizer.encode(text, **encode_kwargs)
+ plan = _plan_for(tokenizer)
+ if plan is None:
+ return tokenizer.encode(text, **encode_kwargs)
+ backend, specials_added, pattern, npattern, intra = plan
+ if encode_kwargs.get("add_special_tokens", True) and specials_added:
+ return tokenizer.encode(text, **encode_kwargs)
+ if (
+ backend.truncation is not None
+ or backend.padding is not None
+ or backend.encode_special_tokens
+ ):
+ return tokenizer.encode(text, **encode_kwargs)
+ chunks = split_prompt(text, pattern, npattern, intra)
+ if len(chunks) < 2:
+ return tokenizer.encode(text, **encode_kwargs)
+
+ check = _parallel_ok is None
+ if check:
+ w0, c0 = time.perf_counter(), time.process_time()
+ encodings = backend.encode_batch(chunks, add_special_tokens=False)
+ if check:
+ wall, cpu = time.perf_counter() - w0, time.process_time() - c0
+ # Rayon worker CPU adds up well beyond wall time. Serial stays near 1.0;
+ # 1.5 leaves headroom for scheduler and measurement noise.
+ _parallel_ok = not (wall > 0.005 and cpu < 1.5 * wall)
+ if not _parallel_ok:
+ logger.warning(
+ "parallel_prompt_encode: tokenizers encode_batch runs serially in "
+ "this process (TOKENIZERS_PARALLELISM off or process forked after "
+ "tokenizers used its pool); using the original encode."
+ )
+ return list(chain.from_iterable(e.ids for e in encodings))
diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py
index 04f30d2f8f..6c19793192 100644
--- a/python/sglang/srt/entrypoints/openai/serving_chat.py
+++ b/python/sglang/srt/entrypoints/openai/serving_chat.py
@@ -46,6 +46,9 @@ from sglang.srt.entrypoints.openai import (
encoding_dsv32,
encoding_dsv41,
)
+from sglang.srt.entrypoints.openai.parallel_prompt_encode import (
+ parallel_prompt_encode,
+)
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionMessageContentTextPart,
ChatCompletionMessageContentVideoPart,
@@ -1836,8 +1839,8 @@ class OpenAIServingChat(OpenAIServingBase):
return_dict=False,
**template_kwargs,
)
- prompt_ids = self.tokenizer_manager.tokenizer.encode(
- rendered_prompt, **encode_kwargs
+ prompt_ids = parallel_prompt_encode(
+ self.tokenizer_manager.tokenizer, rendered_prompt, encode_kwargs
)
decoded_prompt = (
self.tokenizer_manager.tokenizer.decode(prompt_ids)
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 7283e0a280..5befef1ff8 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -1514,6 +1514,11 @@ class Envs:
# Think tokens budget: negative means unlimited, >= 0 caps thinking tokens
SGLANG_MAX_THINK_TOKENS = EnvInt(-1)
SGLANG_PATCH_TOKENIZER = EnvBool(True)
+ # Encode long rendered chat prompts as chunks on the tokenizers thread pool.
+ SGLANG_PARALLEL_PROMPT_ENCODE = EnvBool(True)
+ # Shorter prompts use the single-call encode; below this the gain is eaten
+ # by rayon dispatch and the id merge.
+ SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS = EnvInt(32768)
SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4)
SGLANG_DEFAULT_THINKING = EnvBool(False)
diff --git a/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py b/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py
new file mode 100644
index 0000000000..980e7ac262
--- /dev/null
+++ b/test/registered/unit/entrypoints/openai/test_parallel_prompt_encode.py
@@ -0,0 +1,593 @@
+"""Chunked prompt encoding must produce the ids the single encode produces.
+
+`parallel_prompt_encode` cuts a rendered prompt and sends the pieces through
+`backend.encode_batch`, so its whole contract is that the concatenated ids are
+identical to `tokenizer.encode(text, **encode_kwargs)` — an off-by-one cut
+point is not a crash, it is a silently different prompt. These cases pin the
+identity on a ByteLevel-BPE tokenizer built in memory with the exact
+Split-regex + ByteLevel + BPE pipeline the intra-gap cuts were verified for,
+and pin the refusals: every tokenizer configuration outside that pipeline, and
+every gate, must reach the original encode instead.
+
+The tests never assert that the rayon pool actually ran the chunks in
+parallel. The helper probes for a serial `encode_batch` once and then disables
+itself for the process, which is a correctness-preserving decision a CPU
+runner is free to make; `_parallel_ok` is pinned per test so the id
+comparisons do not depend on it.
+"""
+
+import random
+import sys
+import unittest
+import unittest.mock
+import weakref
+from functools import lru_cache
+
+import pytest
+from tokenizers import (
+ AddedToken,
+ Regex,
+ Tokenizer,
+ decoders,
+ models,
+ normalizers,
+ pre_tokenizers,
+ processors,
+ trainers,
+)
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
+
+from sglang.srt.entrypoints.openai import parallel_prompt_encode as ppe
+from sglang.srt.environ import envs
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=15, suite="base-a-test-cpu")
+
+_TRAIN_CORPUS = [
+ "the quick brown fox jumps over the lazy dog",
+ "hello world, this is a test of the tokenizer",
+ "def main(argv):\n return 0\n",
+ "it's they're I'm we'll you'd she's",
+ "中文文本 日本語 한국어 émigré naïve Ωmega",
+ "numbers 123 4567 89 1,234.56 [bracket]",
+]
+_ALL_KWARGS = ({}, {"add_special_tokens": True}, {"add_special_tokens": False})
+_SPECIALS = [
+ "<|user|>",
+ "<|assistant|>",
+ "",
+ "",
+ "",
+ "[gMASK]",
+ "",
+ "/nothink",
+]
+
+
+@lru_cache(maxsize=1)
+def _safe_backend_json():
+ """A GLM / Llama-3 shaped backend: the pipeline `_intra_cut_safe` accepts.
+
+ Only the pre-tokenizer pipeline and the model type decide which cut points
+ the helper takes, so a 900-token vocabulary exercises the same code path a
+ real checkpoint does. The ByteLevel alphabet is seeded explicitly so every
+ byte of the test corpus has a token and nothing is silently dropped.
+ """
+ backend = Tokenizer(models.BPE())
+ backend.pre_tokenizer = pre_tokenizers.Sequence(
+ [
+ pre_tokenizers.Split(
+ Regex(ppe._INTRA_SPLIT_REGEX), behavior="isolated", invert=False
+ ),
+ pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
+ ]
+ )
+ backend.decoder = decoders.ByteLevel()
+ backend.train_from_iterator(
+ _TRAIN_CORPUS * 20,
+ trainers.BpeTrainer(
+ vocab_size=900,
+ show_progress=False,
+ initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+ special_tokens=[],
+ ),
+ )
+ backend.add_tokens(
+ [AddedToken(t, normalized=False, special=True) for t in _SPECIALS]
+ )
+ return backend.to_str()
+
+
+def _tokenizer(mutate=None, **kwargs):
+ backend = Tokenizer.from_str(_safe_backend_json())
+ if mutate is not None:
+ mutate(backend)
+ return PreTrainedTokenizerFast(tokenizer_object=backend, **kwargs)
+
+
+def _chunk_ids(tokenizer, text, all_cuts=False):
+ """Encode `text` chunk by chunk, the way the helper splits it.
+
+ Each chunk goes through the backend on this thread, so the comparison is
+ the plan's cut points against a single encode — not `encode_batch`'s
+ threading.
+ """
+ backend, _, pattern, npattern, intra = ppe._plan_for(tokenizer)
+ if all_cuts:
+ # Target size 1: take every candidate cut point, not one per 2 KiB.
+ with unittest.mock.patch.object(ppe, "_MIN_CHUNK_CHARS", 1):
+ chunks = ppe.split_prompt(
+ text, pattern, npattern, intra, max_chunks=1 << 30
+ )
+ else:
+ chunks = ppe.split_prompt(text, pattern, npattern, intra)
+ ids = []
+ for chunk in chunks:
+ ids.extend(backend.encode(chunk, add_special_tokens=False).ids)
+ return ids, chunks
+
+
+_EDGE_TEXTS = [
+ "",
+ "a",
+ " ",
+ "\n",
+ "abc def",
+ "no specials at all\nline two\r\nline3 foo",
+ "<|user|>",
+ "<|user|><|assistant|>",
+ "<|user|>x<|assistant|>",
+ "<|user|> hello world",
+ "hello world <|assistant|>",
+ "[gMASK]<|user|>\nhi<|assistant|>\nreasoninganswer",
+ "/nothinking",
+ "/nothinker",
+ "",
+ "<>",
+ "中文文本,没有空格。日本語のテキスト 한국어",
+ "emoji 🙂🙃 x 👩👩👧 é é \u200b x",
+ "tabs\tand double spaces\n\n\nX\n \nY \n Z",
+ "it's they're I'M we'll 'd x's",
+ "\r\nA\rB\n\rC",
+ "123456789 abc123def 1,234.56",
+ "word Line Para",
+ "Ωmega ΣΙΓΜΑ ßtraße İi",
+ "a" * 5000 + " b",
+ "x y\n" * 3000,
+]
+
+
+def _fuzz_texts():
+ """Random texts over the atoms that sit on either side of a cut point."""
+ rng = random.Random(0)
+ atoms = list("aZ é中🙂'1 \n\r\t.,;:{}()<>/_-\"\\") + [
+ "\u0301",
+ " ",
+ "'s",
+ " ",
+ "\r\n",
+ " \n",
+ "\n ",
+ ]
+ atoms += _SPECIALS
+ texts = [
+ "".join(rng.choice(atoms) for _ in range(rng.randint(0, 400)))
+ for _ in range(4000)
+ ]
+ texts += [
+ "".join(rng.choice(atoms) for _ in range(rng.randint(5000, 60000)))
+ for _ in range(30)
+ ]
+ return texts
+
+
+class _EncodeCase(CustomTestCase):
+ def setUp(self):
+ super().setUp()
+ # `_plans` and `_parallel_ok` are process-wide and sticky: a plan
+ # cached for a mutated tokenizer, or a probe that fell back, would
+ # leak into whatever case runs next.
+ self.addCleanup(setattr, ppe, "_plans", ppe._plans)
+ ppe._plans = weakref.WeakKeyDictionary()
+ self.addCleanup(setattr, ppe, "_parallel_ok", ppe._parallel_ok)
+ ppe._parallel_ok = True
+
+ def _set_min_chars(self, value):
+ override = envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.override(value)
+ override.__enter__()
+ self.addCleanup(override.__exit__, None, None, None)
+
+ def _no_min_chars(self):
+ self._set_min_chars(0)
+
+ def _assert_matches_encode(self, tokenizer, text, kwargs_variants=_ALL_KWARGS):
+ for encode_kwargs in kwargs_variants:
+ self.assertEqual(
+ ppe.parallel_prompt_encode(tokenizer, text, encode_kwargs),
+ tokenizer.encode(text, **encode_kwargs),
+ f"{encode_kwargs} on {text[:60]!r}",
+ )
+
+
+class TestSafePipelineIdsAreIdentical(_EncodeCase):
+ def setUp(self):
+ super().setUp()
+ self._no_min_chars()
+ self.tokenizer = _tokenizer()
+
+ def test_the_plan_takes_every_kind_of_cut_point(self):
+ """Without this the id comparisons below could pass on no cuts at all."""
+ plan = ppe._plan_for(self.tokenizer)
+ self.assertIsNotNone(plan)
+ self.assertIsNotNone(plan[2], "added tokens must be matched")
+ self.assertIs(plan[4], ppe._INTRA_CUT, "intra-gap cuts must be enabled")
+ for text, expected in (
+ ("abcd", ["", "abcd"]),
+ ("abcd efgh", ["abcd", " efgh"]),
+ ("abcd\nefgh", ["abcd\n", "efgh"]),
+ ):
+ self.assertEqual(
+ _chunk_ids(self.tokenizer, text, all_cuts=True)[1], expected
+ )
+
+ def test_edge_cases_encode_the_same_at_every_cut_point(self):
+ texts = list(_EDGE_TEXTS)
+ for text in _EDGE_TEXTS:
+ texts += [
+ "" + text,
+ text + "",
+ "" + text + "",
+ ]
+ for text in texts:
+ reference = self.tokenizer.encode(text)
+ self.assertEqual(
+ _chunk_ids(self.tokenizer, text, all_cuts=True)[0], reference
+ )
+ self._assert_matches_encode(self.tokenizer, text)
+
+ def test_fuzzed_texts_encode_the_same_at_every_cut_point(self):
+ for text in _fuzz_texts():
+ reference = self.tokenizer.encode(text)
+ self.assertEqual(
+ _chunk_ids(self.tokenizer, text, all_cuts=True)[0],
+ reference,
+ repr(text[:80]),
+ )
+ self.assertEqual(
+ ppe.parallel_prompt_encode(self.tokenizer, text, {}), reference
+ )
+
+ def test_a_long_prompt_uses_the_default_chunking(self):
+ text = (
+ "<|user|>\n" + "the quick brown fox jumps over the lazy dog\n" * 40
+ ) * 60
+ ids, chunks = _chunk_ids(self.tokenizer, text)
+ self.assertGreater(len(chunks), 1)
+ self.assertLessEqual(len(chunks), ppe._MAX_CHUNKS + 1)
+ self.assertEqual(ids, self.tokenizer.encode(text))
+ self._assert_matches_encode(self.tokenizer, text)
+
+ def test_the_serial_probe_leaves_the_ids_unchanged(self):
+ """The first call times `encode_batch`; whichever way that goes, the
+ ids are the ones `encode` returns."""
+ ppe._parallel_ok = None
+ text = "hello world, this is a test\n" * 4000
+ self.assertEqual(
+ ppe.parallel_prompt_encode(self.tokenizer, text, {}),
+ self.tokenizer.encode(text),
+ )
+ self.assertIn(ppe._parallel_ok, (True, False))
+
+ def _run_probe(self, process_cpu, thread_cpu):
+ ppe._parallel_ok = None
+ text = "hello world, this is a test\n" * 4000
+ with (
+ unittest.mock.patch.object(
+ ppe.time, "perf_counter", side_effect=(1.0, 1.01)
+ ),
+ unittest.mock.patch.object(
+ ppe.time, "thread_time", side_effect=(1.0, 1.0 + thread_cpu)
+ ),
+ unittest.mock.patch.object(
+ ppe.time,
+ "process_time",
+ side_effect=(1.0, 1.0 + process_cpu),
+ create=True,
+ ),
+ ):
+ ppe.parallel_prompt_encode(self.tokenizer, text, {})
+ return ppe._parallel_ok
+
+ def test_the_probe_counts_rayon_worker_cpu(self):
+ self.assertTrue(self._run_probe(process_cpu=0.04, thread_cpu=0.006))
+
+ def test_the_probe_rejects_one_cpu(self):
+ self.assertFalse(self._run_probe(process_cpu=0.01, thread_cpu=0.0))
+
+ def test_a_forked_process_rechecks_parallelism(self):
+ old_pid = getattr(ppe, "_parallel_pid", None)
+ self.addCleanup(setattr, ppe, "_parallel_pid", old_pid)
+ ppe._parallel_ok = False
+ ppe._parallel_pid = 100
+ text = "hello world, this is a test\n" * 4000
+ with (
+ unittest.mock.patch.object(ppe.os, "getpid", return_value=101),
+ unittest.mock.patch.object(
+ ppe.time, "perf_counter", side_effect=(1.0, 1.01)
+ ),
+ unittest.mock.patch.object(
+ ppe.time, "process_time", side_effect=(1.0, 1.04)
+ ),
+ ):
+ ppe.parallel_prompt_encode(self.tokenizer, text, {})
+ self.assertEqual(ppe._parallel_pid, 101)
+ self.assertTrue(ppe._parallel_ok)
+
+
+def _metaspace(backend):
+ backend.pre_tokenizer = pre_tokenizers.Metaspace()
+
+
+def _other_split_regex(backend):
+ backend.pre_tokenizer = pre_tokenizers.Sequence(
+ [
+ pre_tokenizers.Split(Regex(r"\s+|\w+|[^\w\s]+"), behavior="isolated"),
+ pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
+ ]
+ )
+
+
+def _prefix_space(backend):
+ backend.pre_tokenizer = pre_tokenizers.Sequence(
+ [
+ pre_tokenizers.Split(Regex(ppe._INTRA_SPLIT_REGEX), behavior="isolated"),
+ pre_tokenizers.ByteLevel(add_prefix_space=True, use_regex=False),
+ ]
+ )
+
+
+def _lowercase_normalizer(backend):
+ backend.add_tokens([AddedToken("NORM", normalized=True)])
+ backend.normalizer = normalizers.Lowercase()
+
+
+def _post_processor_adds_bos(backend):
+ backend.post_processor = processors.TemplateProcessing(
+ single=" $A",
+ special_tokens=[("", backend.token_to_id(""))],
+ )
+
+
+def _post_processor_duplicates_sequence(backend):
+ backend.post_processor = processors.TemplateProcessing(
+ single="$A $A", special_tokens=[]
+ )
+
+
+def _post_processor_sequence(backend, duplicate=False):
+ template = "$A $A" if duplicate else " $A"
+ backend.post_processor = processors.Sequence(
+ [
+ processors.ByteLevel(),
+ processors.TemplateProcessing(
+ single=template,
+ special_tokens=[("", backend.token_to_id(""))],
+ ),
+ ]
+ )
+
+
+class _WrappedTokenizer(PreTrainedTokenizerFast):
+ def encode(self, text, **kwargs):
+ return super().encode(text, **kwargs)
+
+
+class _SlowWordTokenizer(PreTrainedTokenizer):
+ """A whitespace tokenizer on the python backend, i.e. `is_fast` False."""
+
+ def __init__(self, **kwargs):
+ self._vocab = {
+ w: i for i, w in enumerate(["", "hello", "world", "a", "b"])
+ }
+ super().__init__(unk_token="", **kwargs)
+
+ @property
+ def vocab_size(self):
+ return len(self._vocab)
+
+ def get_vocab(self):
+ return dict(self._vocab)
+
+ def _tokenize(self, text, **kwargs):
+ return text.split()
+
+ def _convert_token_to_id(self, token):
+ return self._vocab.get(token, self._vocab[""])
+
+ def _convert_id_to_token(self, index):
+ return list(self._vocab)[index]
+
+
+class TestUnsafeTokenizersFallBack(_EncodeCase):
+ def setUp(self):
+ super().setUp()
+ self._no_min_chars()
+ self.text = (
+ "Hello NORM world\nsecond line, it's fine tail " * 200
+ )
+
+ def _assert_original_encode(self, tokenizer, kwargs_variants=_ALL_KWARGS):
+ with unittest.mock.patch.object(
+ ppe, "split_prompt", side_effect=AssertionError("chunked path taken")
+ ):
+ self._assert_matches_encode(tokenizer, self.text, kwargs_variants)
+
+ def test_configurations_without_a_plan(self):
+ cases = {
+ "Metaspace pre-tokenizer": _tokenizer(_metaspace),
+ "lstrip added token": _tokenizer(
+ lambda b: b.add_tokens(
+ [AddedToken("", lstrip=True, normalized=False)]
+ )
+ ),
+ "rstrip added token": _tokenizer(
+ lambda b: b.add_tokens(
+ [AddedToken("", rstrip=True, normalized=False)]
+ )
+ ),
+ "single_word added token": _tokenizer(
+ lambda b: b.add_tokens(
+ [AddedToken("", single_word=True, normalized=False)]
+ )
+ ),
+ "split_special_tokens": _tokenizer(split_special_tokens=True),
+ "wrapped encode": _WrappedTokenizer(
+ tokenizer_object=Tokenizer.from_str(_safe_backend_json())
+ ),
+ "slow tokenizer": _SlowWordTokenizer(),
+ }
+ for name, tokenizer in cases.items():
+ with self.subTest(name):
+ self.assertIsNone(ppe._plan_for(tokenizer))
+ self._assert_original_encode(tokenizer)
+
+ def test_configurations_that_keep_only_the_added_token_cuts(self):
+ """A safe-but-unverified pre-tokenizer still cuts at added tokens: the
+ backend restarts the pipeline there whatever the pre-tokenizer is."""
+ cases = {
+ "different Split regex": _tokenizer(_other_split_regex),
+ "ByteLevel add_prefix_space": _tokenizer(_prefix_space),
+ "normalizer present": _tokenizer(_lowercase_normalizer),
+ }
+ for name, tokenizer in cases.items():
+ with self.subTest(name):
+ plan = ppe._plan_for(tokenizer)
+ self.assertIsNotNone(plan)
+ self.assertIsNone(plan[4], "intra-gap cuts are not verified here")
+ self.assertEqual(
+ _chunk_ids(tokenizer, self.text, all_cuts=True)[0],
+ tokenizer.encode(self.text, add_special_tokens=False),
+ )
+ self._assert_matches_encode(tokenizer, self.text)
+
+ def test_a_post_processor_that_adds_specials_falls_back_when_asked_for_them(self):
+ tokenizer = _tokenizer(_post_processor_adds_bos)
+ plan = ppe._plan_for(tokenizer)
+ self.assertIsNotNone(plan)
+ self.assertEqual(plan[1], 1)
+ with unittest.mock.patch.object(
+ ppe, "split_prompt", side_effect=AssertionError("chunked path taken")
+ ):
+ for encode_kwargs in ({}, {"add_special_tokens": True}):
+ self.assertEqual(
+ ppe.parallel_prompt_encode(tokenizer, self.text, encode_kwargs),
+ tokenizer.encode(self.text, **encode_kwargs),
+ )
+ # Asked not to add them, the chunked path is exact again.
+ encode_kwargs = {"add_special_tokens": False}
+ self.assertEqual(
+ ppe.parallel_prompt_encode(tokenizer, self.text, encode_kwargs),
+ tokenizer.encode(self.text, **encode_kwargs),
+ )
+ self.assertEqual(
+ _chunk_ids(tokenizer, self.text, all_cuts=True)[0],
+ tokenizer.encode(self.text, add_special_tokens=False),
+ )
+
+ def test_a_post_processor_that_duplicates_ids_falls_back(self):
+ tok = _tokenizer(_post_processor_duplicates_sequence)
+ text = "hello world " + "hello world " * 2000
+ self.assertIsNone(ppe._plan_for(tok))
+ with unittest.mock.patch.object(
+ ppe, "split_prompt", side_effect=AssertionError("chunked path taken")
+ ):
+ assert ppe.parallel_prompt_encode(
+ tok, text, {"add_special_tokens": False}
+ ) == tok.encode(text, add_special_tokens=False)
+
+ def test_sequence_post_processors_are_checked_recursively(self):
+ safe = _tokenizer(lambda b: _post_processor_sequence(b))
+ unsafe = _tokenizer(lambda b: _post_processor_sequence(b, duplicate=True))
+ self.assertIsNotNone(ppe._plan_for(safe))
+ self.assertIsNone(ppe._plan_for(unsafe))
+ self._assert_original_encode(unsafe)
+
+ def test_backend_state_that_rewrites_the_ids_falls_back(self):
+ """These three live on the backend, not in the plan, so the helper has
+ to look at them per call.
+
+ The mutation is applied after the wrapper is built (it copies the
+ backend), and each case gets one call: falling back runs
+ `tokenizer.encode`, which clears the backend's truncation and padding.
+ """
+ cases = {
+ "truncation": lambda b: b.enable_truncation(50),
+ "padding": lambda b: b.enable_padding(length=64),
+ "encode_special_tokens": lambda b: setattr(
+ b, "encode_special_tokens", True
+ ),
+ }
+ for name, mutate in cases.items():
+ with self.subTest(name):
+ tokenizer = _tokenizer()
+ mutate(tokenizer._tokenizer)
+ self.assertIsNotNone(ppe._plan_for(tokenizer))
+ self._assert_original_encode(tokenizer, ({},))
+
+
+class TestGatesAndArgumentAllowlist(_EncodeCase):
+ threshold = 4096
+
+ def setUp(self):
+ super().setUp()
+ # Pinned rather than inherited: the ambient environment may carry its
+ # own threshold, and the default is 32 KiB of text per case.
+ self._set_min_chars(self.threshold)
+ self.tokenizer = _tokenizer()
+ self.text = "the quick brown fox jumps over the lazy dog\n" * 200
+ self.assertGreater(len(self.text), self.threshold)
+
+ def _assert_chunked(self, encode_kwargs):
+ with unittest.mock.patch.object(
+ ppe, "split_prompt", wraps=ppe.split_prompt
+ ) as split:
+ ids = ppe.parallel_prompt_encode(self.tokenizer, self.text, encode_kwargs)
+ self.assertEqual(split.call_count, 1)
+ self.assertEqual(ids, self.tokenizer.encode(self.text, **(encode_kwargs or {})))
+
+ def _assert_original(self, text, encode_kwargs):
+ with unittest.mock.patch.object(
+ ppe, "split_prompt", side_effect=AssertionError("chunked path taken")
+ ):
+ ids = ppe.parallel_prompt_encode(self.tokenizer, text, encode_kwargs)
+ self.assertEqual(ids, self.tokenizer.encode(text, **(encode_kwargs or {})))
+
+ def test_a_long_prompt_takes_the_chunked_path(self):
+ self._assert_chunked(None)
+ self._assert_chunked({"add_special_tokens": True})
+ self._assert_chunked({"add_special_tokens": False})
+
+ def test_a_prompt_below_the_char_threshold_uses_the_original_encode(self):
+ self._assert_original(self.text[: self.threshold - 1], None)
+ with envs.SGLANG_PARALLEL_PROMPT_ENCODE_MIN_CHARS.override(len(self.text) + 1):
+ self._assert_original(self.text, None)
+
+ def test_the_env_switch_disables_the_chunked_path(self):
+ with envs.SGLANG_PARALLEL_PROMPT_ENCODE.override(False):
+ self._assert_original(self.text, None)
+
+ def test_encode_kwargs_outside_the_allowlist_use_the_original_encode(self):
+ self._assert_original(self.text, {"truncation": False})
+ self._assert_original(
+ self.text, {"add_special_tokens": False, "truncation": False}
+ )
+
+ def test_a_process_with_serial_encode_batch_stays_on_the_original_encode(self):
+ ppe._parallel_ok = False
+ self._assert_original(self.text, None)
+
+
+if __name__ == "__main__":
+ sys.exit(pytest.main([__file__, "-v"]))