Fix /generate endpoint crash when sampling params contain null values (#23401)

This commit is contained in:
cctry
2026-04-22 09:56:10 -07:00
committed by GitHub
parent ac351c1f04
commit 0addd185af
+27 -13
View File
@@ -65,6 +65,8 @@ class SamplingParams:
logit_bias: Optional[Dict[str, float]] = None,
sampling_seed: Optional[int] = None,
) -> None:
# For non-optional params, treat None as "use default" so that callers
# (e.g. /generate) can pass null without crashing verify().
self.max_new_tokens = max_new_tokens
self.stop_strs = stop
if stop_token_ids:
@@ -73,23 +75,35 @@ class SamplingParams:
else:
self.stop_token_ids = None
self.stop_regex_strs = stop_regex
self.temperature = temperature
self.top_p = top_p
self.top_k = top_k
self.min_p = min_p
self.frequency_penalty = frequency_penalty
self.presence_penalty = presence_penalty
self.repetition_penalty = repetition_penalty
self.min_new_tokens = min_new_tokens
self.temperature = temperature if temperature is not None else 1.0
self.top_p = top_p if top_p is not None else 1.0
self.top_k = top_k if top_k is not None else -1
self.min_p = min_p if min_p is not None else 0.0
self.frequency_penalty = (
frequency_penalty if frequency_penalty is not None else 0.0
)
self.presence_penalty = (
presence_penalty if presence_penalty is not None else 0.0
)
self.repetition_penalty = (
repetition_penalty if repetition_penalty is not None else 1.0
)
self.min_new_tokens = min_new_tokens if min_new_tokens is not None else 0
self.regex = regex
self.n = n
self.n = n if n is not None else 1
self.json_schema = json_schema
self.ebnf = ebnf
self.structural_tag = structural_tag
self.ignore_eos = ignore_eos
self.skip_special_tokens = skip_special_tokens
self.spaces_between_special_tokens = spaces_between_special_tokens
self.no_stop_trim = no_stop_trim
self.ignore_eos = ignore_eos if ignore_eos is not None else False
self.skip_special_tokens = (
skip_special_tokens if skip_special_tokens is not None else True
)
self.spaces_between_special_tokens = (
spaces_between_special_tokens
if spaces_between_special_tokens is not None
else True
)
self.no_stop_trim = no_stop_trim if no_stop_trim is not None else False
self.custom_params = custom_params
self.stream_interval = stream_interval
self.logit_bias = logit_bias