[Feature] Beam search support (#31626)
Co-authored-by: cswuyg <cswuyg@gmail.com> Co-authored-by: cswuyg <496090217@qq.com> Co-authored-by: Vedant Jhaveri <vedantjh2@gmail.com> Co-authored-by: Vedant Jhaveri <vjhaveri@linkedin.com>
This commit is contained in:
co-authored by
cswuyg
cswuyg
Vedant Jhaveri
Vedant Jhaveri
parent
e5a1c5a423
commit
ec4bdbfa4a
@@ -68,6 +68,9 @@ class SamplingParams(msgspec.Struct, kw_only=True, array_like=True):
|
||||
repetition_penalty: float = 1.0
|
||||
min_new_tokens: int = 0
|
||||
n: int = 1
|
||||
# beam_width > 1 turns the request into a beam search request; n then means
|
||||
# "number of returned sequences" rather than parallel samples (n <= beam_width).
|
||||
beam_width: Optional[int] = None
|
||||
json_schema: Optional[str] = None
|
||||
regex: Optional[str] = None
|
||||
ebnf: Optional[str] = None
|
||||
@@ -149,6 +152,8 @@ class SamplingParams(msgspec.Struct, kw_only=True, array_like=True):
|
||||
self.top_k = TOP_K_ALL # whole vocabulary
|
||||
|
||||
def verify(self, vocab_size):
|
||||
if self.beam_width is not None and self.beam_width < 1:
|
||||
raise ValueError(f"beam_width must be at least 1, got {self.beam_width}.")
|
||||
if not math.isfinite(self.temperature) or self.temperature < 0.0:
|
||||
raise ValueError(
|
||||
f"temperature must be a non-negative finite number, got {self.temperature}."
|
||||
|
||||
Reference in New Issue
Block a user