[Feature] Beam search support (#31626)

Co-authored-by: cswuyg <cswuyg@gmail.com>
Co-authored-by: cswuyg <496090217@qq.com>
Co-authored-by: Vedant Jhaveri <vedantjh2@gmail.com>
Co-authored-by: Vedant Jhaveri <vjhaveri@linkedin.com>
This commit is contained in:
Liangsheng Yin
2026-08-26 16:56:15 -07:00
committed by GitHub
co-authored by cswuyg cswuyg Vedant Jhaveri Vedant Jhaveri
parent e5a1c5a423
commit ec4bdbfa4a
39 changed files with 3066 additions and 33 deletions
@@ -68,6 +68,9 @@ class SamplingParams(msgspec.Struct, kw_only=True, array_like=True):
repetition_penalty: float = 1.0
min_new_tokens: int = 0
n: int = 1
# beam_width > 1 turns the request into a beam search request; n then means
# "number of returned sequences" rather than parallel samples (n <= beam_width).
beam_width: Optional[int] = None
json_schema: Optional[str] = None
regex: Optional[str] = None
ebnf: Optional[str] = None
@@ -149,6 +152,8 @@ class SamplingParams(msgspec.Struct, kw_only=True, array_like=True):
self.top_k = TOP_K_ALL # whole vocabulary
def verify(self, vocab_size):
if self.beam_width is not None and self.beam_width < 1:
raise ValueError(f"beam_width must be at least 1, got {self.beam_width}.")
if not math.isfinite(self.temperature) or self.temperature < 0.0:
raise ValueError(
f"temperature must be a non-negative finite number, got {self.temperature}."