[Model] Add native IFM K2 Horizon serving support (#37654)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Yash Akhauri
2026-09-03 16:39:43 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang Xinyuan Tong
parent 02d9b3060a
commit 3bac084d4e
34 changed files with 4487 additions and 16 deletions
@@ -27,6 +27,8 @@ const MAX_STOP_REGEX_LEN: usize = 256;
/// Most `stop_regex` patterns accepted per request. Python's `re` cache holds 512
/// (`re._MAXCACHE`), so past that every pattern recompiles on every decode step.
const MAX_STOP_REGEX_COUNT: usize = 32;
const REQUEST_REASONING_END_TOKEN_IDS_KEY: &str = "__sglang_reasoning_end_token_ids";
const MAX_REQUEST_REASONING_END_TOKEN_IDS: usize = 32;
/// JSON values accepted by Python's `CustomParamValue`: a scalar, a list of
/// scalars, or a string-keyed object whose values are scalars.
@@ -584,6 +586,38 @@ impl SamplingParams {
}
}
}
if let Some(value) = self
.custom_params
.as_ref()
.and_then(|params| params.get(REQUEST_REASONING_END_TOKEN_IDS_KEY))
{
let CustomParamValue::List(token_ids) = value else {
return Err(bad(
"request reasoning end token IDs must be a list of integers".into(),
));
};
if token_ids.is_empty() || token_ids.len() > MAX_REQUEST_REASONING_END_TOKEN_IDS {
return Err(bad(format!(
"request reasoning end token IDs must contain 1 to \
{MAX_REQUEST_REASONING_END_TOKEN_IDS} integers"
)));
}
for token_id in token_ids {
let in_vocab = match token_id {
JsonScalar::Signed(token_id) => {
*token_id >= 0 && (*token_id as u64) < vocab_size
}
JsonScalar::Unsigned(token_id) => *token_id < vocab_size,
_ => false,
};
if !in_vocab {
return Err(bad(format!(
"request reasoning end token IDs must be integers in [0, {})",
vocab_size
)));
}
}
}
// Grammars are mutually exclusive.
let grammars = [
&self.json_schema,
@@ -1057,6 +1091,27 @@ mod tests {
}
}
#[test]
fn request_reasoning_end_token_ids_are_bounded_integers() {
let valid = norm(r#"{"custom_params":{"__sglang_reasoning_end_token_ids":[17,18]}}"#);
assert!(valid.custom_params.is_some());
for body in [
r#"{"custom_params":{"__sglang_reasoning_end_token_ids":[]}}"#,
r#"{"custom_params":{"__sglang_reasoning_end_token_ids":[-1]}}"#,
r#"{"custom_params":{"__sglang_reasoning_end_token_ids":[true]}}"#,
r#"{"custom_params":{"__sglang_reasoning_end_token_ids":[32000]}}"#,
r#"{"custom_params":{"__sglang_reasoning_end_token_ids":"17"}}"#,
] {
assert!(
serde_json::from_str::<SamplingParams>(body)
.unwrap()
.normalize(false, 32_000)
.is_err()
);
}
}
/// `skip_tokenizer_init` has no tokenizer, so the text-matching stop features
/// and `min_new_tokens` (needs eos_token_id) are 400s, not silent no-ops.
/// Mirrors Python `raise_if_tokenizer_required`.